init
This commit is contained in:
commit
d295c7f139
1527 files changed
+270726
No files matched your search
@@ -0,0 +1,137 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// 4 Day Week provider — board-wide aggregator of 4-day-week / reduced-hours
|
||||
// roles. Public, zero-auth JSON API: https://4dayweek.io/api/jobs
|
||||
// Response shape: { jobs: [ { id, title, slug, company_name, company: { name,
|
||||
// slug, ... }, work_arrangement, locations: [{ city, country, ... }],
|
||||
// posted (epoch SECONDS), is_expired, ... } ], total, page, has_more }
|
||||
//
|
||||
// The feed carries no per-job URL — the canonical posting page is
|
||||
// https://4dayweek.io/job/<slug> (verified via the page's rel=canonical; the
|
||||
// /jobs and /remote-jobs paths soft-404 to the listing). The slug is the only
|
||||
// variable part of the built URL and is constrained to a safe token.
|
||||
//
|
||||
// Paginated 25/page via ?page=N; bounded by max_pages (default 3, cap 50) and
|
||||
// the has_more flag. Expired postings (is_expired) are dropped.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: 4dayweek`.
|
||||
|
||||
const FEED_BASE = 'https://4dayweek.io/api/jobs';
|
||||
const TRUSTED_HOST = '4dayweek.io';
|
||||
const JOB_BASE = `https://${TRUSTED_HOST}/job`;
|
||||
const PER_PAGE = 25;
|
||||
const DEFAULT_MAX_PAGES = 3;
|
||||
const MAX_PAGES_CAP = 50;
|
||||
const SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9-]*$/;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertFourDayUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`4dayweek: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`4dayweek: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST) {
|
||||
throw new Error(`4dayweek: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
|
||||
function resolveMaxPages(entry) {
|
||||
const v = entry?.max_pages;
|
||||
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
|
||||
return DEFAULT_MAX_PAGES;
|
||||
}
|
||||
|
||||
// NaN-safe: posted is epoch SECONDS → ms; anything non-finite yields undefined.
|
||||
function toEpochMs(seconds) {
|
||||
return Number.isFinite(seconds) ? seconds * 1000 : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a single 4 Day Week job. Exported for unit tests.
|
||||
*
|
||||
* Field mapping → the normalized Job shape:
|
||||
* - title: `title`, trimmed (postings without one are dropped).
|
||||
* - url: built as `https://4dayweek.io/job/<slug>` — the feed has no URL
|
||||
* field. The slug must match a safe token (no slashes/spaces) or
|
||||
* the posting is dropped; the URL is the dedup key and display-only.
|
||||
* - company: `company_name` (else `company.name`), falling back to the portal
|
||||
* entry name, then "4 Day Week".
|
||||
* - location: first `locations[]` as "city, country"; "Remote" is appended when
|
||||
* the job's (or location's) work_arrangement is "remote".
|
||||
* - postedAt: `posted` (epoch SECONDS) → epoch ms (omitted when absent).
|
||||
*
|
||||
* Expired postings (`is_expired === true`) are dropped.
|
||||
*
|
||||
* @param {any} j
|
||||
* @param {string} [fallbackCompany]
|
||||
* @returns {{ title: string, url: string, company: string, location: string, postedAt?: number } | null}
|
||||
*/
|
||||
export function normalize4dwJob(j, fallbackCompany) {
|
||||
if (!j || typeof j !== 'object') return null;
|
||||
if (j.is_expired === true) return null;
|
||||
|
||||
const title = typeof j.title === 'string' ? j.title.trim() : '';
|
||||
if (!title) return null;
|
||||
|
||||
const slug = typeof j.slug === 'string' ? j.slug.trim() : '';
|
||||
if (!SLUG_RE.test(slug)) return null; // need a clean slug to build the url
|
||||
const url = `${JOB_BASE}/${encodeURIComponent(slug)}`;
|
||||
|
||||
const company =
|
||||
typeof j.company_name === 'string' && j.company_name.trim()
|
||||
? j.company_name.trim()
|
||||
: j.company && typeof j.company === 'object' && typeof j.company.name === 'string' && j.company.name.trim()
|
||||
? j.company.name.trim()
|
||||
: typeof fallbackCompany === 'string' && fallbackCompany.trim()
|
||||
? fallbackCompany.trim()
|
||||
: '4 Day Week';
|
||||
|
||||
const first = Array.isArray(j.locations) && j.locations[0] && typeof j.locations[0] === 'object' ? j.locations[0] : {};
|
||||
const city = typeof first.city === 'string' ? first.city.trim() : '';
|
||||
const country = typeof first.country === 'string' ? first.country.trim() : '';
|
||||
const base = [city, country].filter(Boolean).join(', ');
|
||||
const remote = j.work_arrangement === 'remote' || first.work_arrangement === 'remote';
|
||||
const location = [base, remote ? 'Remote' : ''].filter(Boolean).join(', ');
|
||||
|
||||
/** @type {{ title: string, url: string, company: string, location: string, postedAt?: number }} */
|
||||
const job = { title, url, company, location };
|
||||
const postedAt = toEpochMs(j.posted);
|
||||
if (postedAt !== undefined) job.postedAt = postedAt;
|
||||
return job;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: '4dayweek',
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
assertFourDayUrl(FEED_BASE);
|
||||
const maxPages = resolveMaxPages(entry);
|
||||
const fallbackCompany = entry?.name;
|
||||
const out = [];
|
||||
|
||||
for (let page = 1; page <= maxPages; page++) {
|
||||
const url = `${FEED_BASE}?page=${page}`;
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(url, { redirect: 'error' });
|
||||
if (!json || !Array.isArray(json.jobs)) {
|
||||
throw new Error(
|
||||
`4dayweek: unexpected API response on page ${page} — expected { jobs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`,
|
||||
);
|
||||
}
|
||||
for (const j of json.jobs) {
|
||||
const normalized = normalize4dwJob(j, fallbackCompany);
|
||||
if (normalized) out.push(normalized);
|
||||
}
|
||||
if (json.has_more === false) break; // last page per the API flag
|
||||
if (json.jobs.length < PER_PAGE) break; // short page → last page
|
||||
}
|
||||
return out;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,53 @@
|
||||
// HTTP transport helpers shared across providers.
|
||||
// Files prefixed with _ are never loaded as providers by scan.mjs.
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 10_000;
|
||||
const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; career-ops/1.3)';
|
||||
|
||||
async function fetchWithTimeout(url, { timeoutMs = DEFAULT_TIMEOUT_MS, headers = {}, method = 'GET', body = null, redirect = 'follow' } = {}) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
method,
|
||||
headers: { 'user-agent': DEFAULT_USER_AGENT, ...headers },
|
||||
body,
|
||||
redirect,
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!res.ok) {
|
||||
const responseText = await res.text().catch(() => '');
|
||||
// WAF/CDN challenge pages (seen live: Workday 429s) carry no actionable
|
||||
// text — HTML markup or a generic interstitial message, not worth
|
||||
// parsing or displaying. The status code and its standard reason
|
||||
// phrase are what a log line needs; the raw body is still attached as
|
||||
// err.body for callers that want to inspect it.
|
||||
const err = new Error(`HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ''}`);
|
||||
err.status = res.status;
|
||||
err.body = responseText;
|
||||
err.retryAfter = res.headers.get('retry-after');
|
||||
throw err;
|
||||
}
|
||||
return res;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchJson(url, opts = {}) {
|
||||
const res = await fetchWithTimeout(url, opts);
|
||||
return await res.json();
|
||||
}
|
||||
|
||||
export async function fetchText(url, opts = {}) {
|
||||
const res = await fetchWithTimeout(url, opts);
|
||||
return await res.text();
|
||||
}
|
||||
|
||||
export function makeHttpCtx() {
|
||||
return {
|
||||
transport: 'http',
|
||||
fetchJson,
|
||||
fetchText,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
// Provider registry — loading and routing shared by the scanner and the portal
|
||||
// health check. Files prefixed with _ are never loaded as providers by scan.mjs,
|
||||
// so this helper module lives safely alongside the provider plugins.
|
||||
//
|
||||
// Extracted from scan.mjs (#1451-era inline definitions) so verify-portals.mjs
|
||||
// can route non-ATS boards through the SAME provider layer the scanner uses,
|
||||
// without importing scan.mjs itself (which has top-level side effects and would
|
||||
// form an import cycle via classifyFetchError).
|
||||
|
||||
import { existsSync, readdirSync } from 'fs';
|
||||
import path from 'path';
|
||||
import { pathToFileURL } from 'url';
|
||||
|
||||
/**
|
||||
* Load every provider plugin in a directory into an id→provider Map.
|
||||
*
|
||||
* Alphabetical order so detect() priority is deterministic across machines.
|
||||
* Malformed modules (wrong shape, duplicate id, import error) are logged and
|
||||
* skipped, never fatal.
|
||||
*
|
||||
* @param {string} dir - Absolute path to the providers directory.
|
||||
* @returns {Promise<Map<string, object>>}
|
||||
*/
|
||||
export async function loadProviders(dir) {
|
||||
const providers = new Map();
|
||||
if (!existsSync(dir)) return providers;
|
||||
const entries = readdirSync(dir)
|
||||
.filter(f => f.endsWith('.mjs') && !f.startsWith('_'))
|
||||
.sort();
|
||||
for (const file of entries) {
|
||||
const full = path.join(dir, file);
|
||||
let mod;
|
||||
try {
|
||||
mod = await import(pathToFileURL(full).href);
|
||||
} catch (err) {
|
||||
console.error(`⚠️ ${file}: failed to load — ${err.message}`);
|
||||
continue;
|
||||
}
|
||||
const p = mod.default;
|
||||
if (!p || typeof p.fetch !== 'function' || !p.id) {
|
||||
console.error(`⚠️ ${file}: skipping — default export must be { id, fetch }`);
|
||||
continue;
|
||||
}
|
||||
if (providers.has(p.id)) {
|
||||
console.error(`⚠️ ${file}: duplicate provider id "${p.id}" — keeping first`);
|
||||
continue;
|
||||
}
|
||||
providers.set(p.id, p);
|
||||
}
|
||||
return providers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve which provider handles a tracked_companies entry.
|
||||
* 1. Explicit `provider:` field wins (skips detect()).
|
||||
* 2. local-parser when parser.command + script are configured (before API detect).
|
||||
* 3. Otherwise each provider's detect() runs in load order; first hit wins.
|
||||
*
|
||||
* @param {object} entry - tracked_companies entry.
|
||||
* @param {Map<string, object>} providers - id→provider Map from loadProviders().
|
||||
* @param {{skipIds?: string[]}} [opts] - Provider ids to skip (e.g. 'local-parser'
|
||||
* so a network-only health check never execs a configured local command).
|
||||
* @returns {{provider: object}|{error: string}|null}
|
||||
*/
|
||||
export function resolveProvider(entry, providers, { skipIds = [] } = {}) {
|
||||
if (entry.provider) {
|
||||
const p = providers.get(entry.provider);
|
||||
if (!p) return { error: `unknown provider: ${entry.provider}` };
|
||||
return { provider: p };
|
||||
}
|
||||
|
||||
const localParser = providers.get('local-parser');
|
||||
if (localParser && !skipIds.includes('local-parser')) {
|
||||
try {
|
||||
const hit = localParser.detect?.(entry);
|
||||
if (hit) return { provider: localParser };
|
||||
} catch (err) {
|
||||
console.error(`⚠️ local-parser: detect() threw for "${entry.name}" — ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const p of providers.values()) {
|
||||
if (skipIds.includes(p.id)) continue;
|
||||
let hit;
|
||||
try {
|
||||
hit = p.detect?.(entry);
|
||||
} catch (err) {
|
||||
console.error(`⚠️ ${p.id}: detect() threw for "${entry.name}" — ${err.message}`);
|
||||
continue;
|
||||
}
|
||||
if (hit) return { provider: p };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* _trust-validator.mjs — Lightweight trust validation for scanned job postings.
|
||||
*
|
||||
* Enriches each job with trustScore (0-100), trustFlags (string[]), and
|
||||
* trustLevel ('high' | 'medium' | 'low'). Never drops jobs — flag only.
|
||||
*
|
||||
* Underscore prefix keeps scan.mjs's provider loader from importing this
|
||||
* as a job-board provider.
|
||||
*
|
||||
* V1 heuristics:
|
||||
* 1. URL structure validation
|
||||
* 2. Missing application URL
|
||||
* 3. Suspicious domain detection
|
||||
* 4. Company ↔ domain mismatch (with ATS allowlist)
|
||||
*/
|
||||
|
||||
/** @type {string[]} */
|
||||
const DEFAULT_SUSPICIOUS_DOMAINS = [
|
||||
'bit.ly',
|
||||
'tinyurl.com',
|
||||
't.co',
|
||||
'forms.gle',
|
||||
'goo.gl',
|
||||
'shorturl.at',
|
||||
'rebrand.ly',
|
||||
'cutt.ly',
|
||||
];
|
||||
|
||||
/** @type {string[]} */
|
||||
const DEFAULT_ATS_ALLOWLIST = [
|
||||
'greenhouse.io',
|
||||
'ashbyhq.com',
|
||||
'lever.co',
|
||||
'workday.com',
|
||||
'smartrecruiters.com',
|
||||
'jobvite.com',
|
||||
'myworkdayjobs.com',
|
||||
'recruitee.com',
|
||||
'workable.com',
|
||||
'icims.com',
|
||||
'taleo.net',
|
||||
'applytojob.com',
|
||||
'breezy.hr',
|
||||
'jazz.co',
|
||||
'bamboohr.com',
|
||||
'teamtailor.com',
|
||||
];
|
||||
|
||||
/** @type {Record<string, number>} */
|
||||
const PENALTIES = {
|
||||
invalid_url: 50,
|
||||
missing_apply_url: 40,
|
||||
suspicious_domain: 25,
|
||||
company_domain_mismatch: 15,
|
||||
};
|
||||
|
||||
/**
|
||||
* Classify a numeric trust score into a human-readable level.
|
||||
*
|
||||
* @param {number} score
|
||||
* @returns {'high' | 'medium' | 'low'}
|
||||
*/
|
||||
export function classifyTrustLevel(score) {
|
||||
if (score >= 90) return 'high';
|
||||
if (score >= 60) return 'medium';
|
||||
return 'low';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a URL string is well-formed and uses http(s).
|
||||
*
|
||||
* @param {string} url
|
||||
* @returns {{ valid: boolean, flag?: string }}
|
||||
*/
|
||||
export function validateUrl(url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
||||
return { valid: false, flag: 'invalid_url' };
|
||||
}
|
||||
return { valid: true };
|
||||
} catch {
|
||||
return { valid: false, flag: 'invalid_url' };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a hostname matches or is a subdomain of any entry in a
|
||||
* domain list. E.g. hostname "abc.bit.ly" matches blocklist entry "bit.ly".
|
||||
*
|
||||
* @param {string} hostname — lowercased hostname from URL
|
||||
* @param {string[]} domainList
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function matchesDomainList(hostname, domainList) {
|
||||
for (const domain of domainList) {
|
||||
if (hostname === domain || hostname.endsWith('.' + domain)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Heuristic: does the company name plausibly match the URL hostname?
|
||||
*
|
||||
* Strips non-alphanumeric chars from the company name, lowercases, and checks
|
||||
* whether the result (or any word ≥3 chars) appears as a substring of the
|
||||
* hostname. Returns true when a match is found (= no mismatch flag needed).
|
||||
*
|
||||
* @param {string} company
|
||||
* @param {string} hostname
|
||||
* @returns {boolean} true if company matches hostname
|
||||
*/
|
||||
export function companyMatchesHostname(company, hostname) {
|
||||
if (!company || !hostname) return true; // can't evaluate → no flag
|
||||
|
||||
const normalized = company.toLowerCase().replace(/[^a-z0-9 ]/g, '').trim();
|
||||
if (!normalized) return true;
|
||||
|
||||
// Full slug check (all spaces removed)
|
||||
const slug = normalized.replace(/\s+/g, '');
|
||||
if (hostname.includes(slug)) return true;
|
||||
|
||||
// Word-level check — any word ≥3 chars present in hostname
|
||||
const words = normalized.split(/\s+/).filter(w => w.length >= 3);
|
||||
for (const word of words) {
|
||||
if (hostname.includes(word)) return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a trust validator function from portals.yml config.
|
||||
*
|
||||
* Usage mirrors buildLocationFilter / buildSalaryFilter in scan.mjs.
|
||||
*
|
||||
* @param {object} [config]
|
||||
* @param {boolean} [config.enabled]
|
||||
* @param {string[]} [config.suspicious_domains]
|
||||
* @param {string[]} [config.ats_allowlist]
|
||||
* @returns {(job: { url?: string, company?: string }) => { score: number, flags: string[], level: 'high' | 'medium' | 'low' }}
|
||||
*/
|
||||
export function buildTrustValidator(config) {
|
||||
// Disabled or absent config → return a no-op that gives every job 100/high
|
||||
if (!config || config.enabled === false) {
|
||||
return () => ({ score: 100, flags: [], level: /** @type {const} */ ('high') });
|
||||
}
|
||||
|
||||
const suspiciousDomains = (Array.isArray(config.suspicious_domains)
|
||||
? config.suspicious_domains
|
||||
: DEFAULT_SUSPICIOUS_DOMAINS)
|
||||
.map(d => String(d).toLowerCase().trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const atsAllowlist = (Array.isArray(config.ats_allowlist)
|
||||
? config.ats_allowlist
|
||||
: DEFAULT_ATS_ALLOWLIST)
|
||||
.map(d => String(d).toLowerCase().trim())
|
||||
.filter(Boolean);
|
||||
|
||||
return (job) => {
|
||||
/** @type {string[]} */
|
||||
const flags = [];
|
||||
let score = 100;
|
||||
|
||||
const url = typeof job.url === 'string' ? job.url.trim() : '';
|
||||
|
||||
// Rule 1 — Missing URL
|
||||
if (!url) {
|
||||
flags.push('missing_apply_url');
|
||||
score -= PENALTIES.missing_apply_url;
|
||||
// Can't run further URL-based checks
|
||||
const clamped = Math.max(0, score);
|
||||
return { score: clamped, flags, level: classifyTrustLevel(clamped) };
|
||||
}
|
||||
|
||||
// Rule 2 — URL structure validation
|
||||
const urlCheck = validateUrl(url);
|
||||
if (!urlCheck.valid) {
|
||||
flags.push('invalid_url');
|
||||
score -= PENALTIES.invalid_url;
|
||||
// Can't parse hostname → skip domain checks
|
||||
const clamped = Math.max(0, score);
|
||||
return { score: clamped, flags, level: classifyTrustLevel(clamped) };
|
||||
}
|
||||
|
||||
// Parse hostname for domain checks
|
||||
let hostname = '';
|
||||
try {
|
||||
hostname = new URL(url).hostname.toLowerCase();
|
||||
} catch {
|
||||
// Already validated above, but guard defensively
|
||||
const clamped = Math.max(0, score);
|
||||
return { score: clamped, flags, level: classifyTrustLevel(clamped) };
|
||||
}
|
||||
|
||||
// Rule 3 — Suspicious domain detection
|
||||
if (matchesDomainList(hostname, suspiciousDomains)) {
|
||||
flags.push('suspicious_domain');
|
||||
score -= PENALTIES.suspicious_domain;
|
||||
}
|
||||
|
||||
// Rule 4 — Company ↔ domain mismatch (skip for ATS-hosted URLs)
|
||||
const company = typeof job.company === 'string' ? job.company.trim() : '';
|
||||
if (company && !matchesDomainList(hostname, atsAllowlist)) {
|
||||
if (!companyMatchesHostname(company, hostname)) {
|
||||
flags.push('company_domain_mismatch');
|
||||
score -= PENALTIES.company_domain_mismatch;
|
||||
}
|
||||
}
|
||||
|
||||
score = Math.max(0, Math.min(100, score));
|
||||
return { score, flags, level: classifyTrustLevel(score) };
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
// Type catalog for the provider plugin contract.
|
||||
//
|
||||
// This file is documentation-only — pure JSDoc @typedef annotations. The
|
||||
// project is plain ESM JavaScript with no build step; provider authors can
|
||||
// reference these types via `/** @typedef {import('./_types.js').Provider} Provider */`
|
||||
// at the top of a `// @ts-check`-enabled file to get IDE hints. The runtime
|
||||
// contract is enforced by scan.mjs (id presence, fetch is a function, fetch
|
||||
// returns an array), not by these annotations.
|
||||
//
|
||||
// Files prefixed with _ are never loaded as providers by scan.mjs.
|
||||
|
||||
/**
|
||||
* Normalized job posting — the unit of currency throughout the scanner.
|
||||
*
|
||||
* @typedef {object} Job
|
||||
* @property {string} title Required, non-empty after trim.
|
||||
* @property {string} url Required, absolute URL — used as the dedup key.
|
||||
* @property {string} company May be empty when the source can't expose it
|
||||
* at the list-page level; populated downstream.
|
||||
* @property {string} location May be empty.
|
||||
* @property {string} [description] Job description text, populated ONLY when the
|
||||
* provider's list payload carries it for free (no
|
||||
* extra per-job request — the scanner is zero-token).
|
||||
* Lever supplies it via `descriptionPlain`; most
|
||||
* providers omit it. Consumed by scan.mjs's
|
||||
* content_filter; an empty/absent value always
|
||||
* passes the filter.
|
||||
* @property {number} [postedAt] Epoch ms when the posting was published.
|
||||
* Omitted when the source doesn't expose a
|
||||
* usable date. scan.mjs ignores it; consumers
|
||||
* like scan-ats-full.mjs use it for recency
|
||||
* filtering.
|
||||
* @property {number} [trustScore] 0-100 trust score from _trust-validator.mjs.
|
||||
* @property {string[]} [trustFlags] Flags raised by trust validation (e.g.
|
||||
* 'invalid_url', 'suspicious_domain').
|
||||
* @property {'high'|'medium'|'low'} [trustLevel] Classification derived from
|
||||
* trustScore.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Result returned by the trust validator for a single job posting.
|
||||
*
|
||||
* @typedef {object} TrustResult
|
||||
* @property {number} score 0-100, where 100 = fully trusted.
|
||||
* @property {string[]} flags Flags raised (e.g. 'invalid_url', 'suspicious_domain').
|
||||
* @property {'high'|'medium'|'low'} level Classification: 90-100 high, 60-89 medium, 0-59 low.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A single `tracked_companies` entry from `portals.yml`.
|
||||
*
|
||||
* Provider-specific fields are opaque to scan.mjs and validated by the
|
||||
* provider itself. Examples in current providers: `api`, `careers_url`.
|
||||
* Providers read these directly off the entry object — no schema enforcement
|
||||
* at the framework level.
|
||||
*
|
||||
* @typedef {object} PortalEntry
|
||||
* @property {string} name User-facing label; appears in logs and placeholders.
|
||||
* @property {boolean} [enabled] Default: true.
|
||||
* @property {string} [careers_url] Public listing URL; consumed by detect().
|
||||
* @property {string} [api] JSON API URL; used directly by greenhouse/ashby providers.
|
||||
* @property {string} [provider] Explicit provider id — bypasses detect().
|
||||
* @property {('http')} [transport] Default: 'http'. Reserved for future transports.
|
||||
* @property {number} [max_pages] Provider-specific pagination cap (avature, workday).
|
||||
* @property {string} [offset_param] avature only: pins the pagination query key and disables the
|
||||
* provider's jobOffset→offset self-heal. Rarely needed — an
|
||||
* escape hatch for a tenant the auto-switch can't resolve.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Returned by `detect()` when a provider claims an entry. `url` is
|
||||
* informational (used in logs); routing only checks for a non-null return.
|
||||
*
|
||||
* @typedef {object} DetectHit
|
||||
* @property {string} url
|
||||
*/
|
||||
|
||||
/**
|
||||
* Options forwarded to the underlying `fetch` call.
|
||||
*
|
||||
* @typedef {object} FetchOptions
|
||||
* @property {number} [timeoutMs]
|
||||
* @property {Object<string,string>} [headers]
|
||||
* @property {string} [method]
|
||||
* @property {(string|null)} [body]
|
||||
* @property {('error'|'follow'|'manual')} [redirect]
|
||||
*/
|
||||
|
||||
/**
|
||||
* What scan.mjs hands to provider.fetch(). For Phase A only `transport: 'http'`
|
||||
* is implemented; the shape reserves room for future transports without
|
||||
* breaking the contract.
|
||||
*
|
||||
* @typedef {object} Context
|
||||
* @property {('http')} transport
|
||||
* @property {(url: string, opts?: FetchOptions) => Promise<string>} fetchText
|
||||
* @property {(url: string, opts?: FetchOptions) => Promise<unknown>} fetchJson
|
||||
* @property {number} [maxPages] Optional pagination hint. When set (verify-portals.mjs's
|
||||
* health probe passes 1), a paginating provider SHOULD stop
|
||||
* after this many pages — the probe only needs the first page
|
||||
* to tell a live board from a broken one, and must not walk an
|
||||
* entire careers site. Providers that ignore it stay correct:
|
||||
* the probe caps their requests defensively via the context's
|
||||
* own fetch functions.
|
||||
* @property {(ms: number) => Promise<void>} [sleep] Optional cross-provider pacing hook used by
|
||||
* paginating providers (avature, workday) to throttle between page
|
||||
* requests. May be absent — providers fall back to a native
|
||||
* `setTimeout`-based delay.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The provider contract — the default export of every providers/*.mjs file
|
||||
* (excluding _-prefixed shared helpers).
|
||||
*
|
||||
* @typedef {object} Provider
|
||||
* @property {string} id Unique across all loaded providers.
|
||||
* @property {((entry: PortalEntry) => (DetectHit | null))} [detect] Optional auto-detection.
|
||||
* @property {(entry: PortalEntry, ctx: Context) => Promise<Job[]>} fetch Required.
|
||||
*/
|
||||
|
||||
export {};
|
||||
@@ -0,0 +1,107 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Amazon / AWS provider — hits the public amazon.jobs search JSON API.
|
||||
// Auto-detects from a careers_url on `amazon.jobs`; the whole board is one
|
||||
// global endpoint, so a tracked_companies entry narrows it with an `amazon:`
|
||||
// config block whose keys are passed straight through as query params:
|
||||
//
|
||||
// - name: Amazon / AWS
|
||||
// provider: amazon
|
||||
// amazon:
|
||||
// loc_query: Germany # free-text location filter
|
||||
// base_query: machine learning # optional keyword filter
|
||||
// category: software-development # optional facet (repeatable via array)
|
||||
//
|
||||
// The board is enormous (100k+ postings), so a location and/or keyword filter
|
||||
// is effectively required — without one the MAX_PAGES cap just returns the most
|
||||
// recent slice. result_limit is fixed at 100 (the API's hard per-page max;
|
||||
// larger values return an empty `jobs`), and we page via `offset`.
|
||||
|
||||
const PAGE_SIZE = 100; // amazon.jobs caps result_limit at 100
|
||||
const MAX_PAGES = 20; // safety cap — at most 2000 postings per entry
|
||||
const ORIGIN = 'https://www.amazon.jobs';
|
||||
|
||||
/** @param {import('./_types.js').PortalEntry & {amazon?: Record<string, unknown>}} entry */
|
||||
function buildQuery(entry, offset) {
|
||||
const cfg = entry.amazon && typeof entry.amazon === 'object' ? entry.amazon : {};
|
||||
const params = new URLSearchParams();
|
||||
// Pass config keys through verbatim (base_query, loc_query, category, …).
|
||||
// Array values are amazon.jobs facet filters and MUST use the `key[]=`
|
||||
// bracket form to filter (e.g. normalized_country_code[]=DEU); a bare
|
||||
// `key=DEU` is silently ignored and the board stays global.
|
||||
for (const [k, v] of Object.entries(cfg)) {
|
||||
if (v == null) continue;
|
||||
if (Array.isArray(v)) {
|
||||
const name = k.endsWith('[]') ? k : `${k}[]`;
|
||||
for (const item of v) params.append(name, String(item));
|
||||
} else {
|
||||
params.append(k, String(v));
|
||||
}
|
||||
}
|
||||
if (!params.has('base_query')) params.set('base_query', '');
|
||||
if (!params.has('loc_query')) params.set('loc_query', '');
|
||||
params.set('sort', cfg.sort ? String(cfg.sort) : 'recent');
|
||||
params.set('result_limit', String(PAGE_SIZE));
|
||||
params.set('offset', String(offset));
|
||||
return `${ORIGIN}/en/search.json?${params.toString()}`;
|
||||
}
|
||||
|
||||
// amazon.jobs posted_date reads "July 3, 2026" (note the padded day);
|
||||
// Date.parse handles it once whitespace is collapsed. (updated_time is a
|
||||
// relative string like "10 minutes" / "about 1 hour" — unparseable, skip it.)
|
||||
function toEpochMs(job) {
|
||||
const raw = job.posted_date;
|
||||
if (!raw || typeof raw !== 'string') return undefined;
|
||||
const parsed = Date.parse(raw.replace(/\s+/g, ' ').trim());
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'amazon',
|
||||
|
||||
detect(entry) {
|
||||
const url = entry.api || entry.careers_url || '';
|
||||
if (typeof url !== 'string') return null;
|
||||
// Match the host, not a path segment, to avoid spoofed URLs.
|
||||
try {
|
||||
const host = new URL(url).host.toLowerCase();
|
||||
if (host === 'amazon.jobs' || host.endsWith('.amazon.jobs')) return { url };
|
||||
} catch {
|
||||
/* not an absolute URL */
|
||||
}
|
||||
return null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const jobs = [];
|
||||
const seen = new Set();
|
||||
for (let page = 0; page < MAX_PAGES; page++) {
|
||||
const url = buildQuery(entry, page * PAGE_SIZE);
|
||||
const json = /** @type {any} */ (await ctx.fetchJson(url, { redirect: 'error' }));
|
||||
const postings = Array.isArray(json?.jobs) ? json.jobs : [];
|
||||
if (postings.length === 0) break;
|
||||
|
||||
let fresh = 0;
|
||||
for (const j of postings) {
|
||||
const path = j.job_path;
|
||||
if (!path || typeof path !== 'string') continue;
|
||||
const url2 = /^https?:\/\//i.test(path) ? path : ORIGIN + (path.startsWith('/') ? path : '/' + path);
|
||||
if (seen.has(url2)) continue;
|
||||
seen.add(url2);
|
||||
fresh++;
|
||||
jobs.push({
|
||||
title: (j.title || '').trim(),
|
||||
url: url2,
|
||||
company: j.company_name || entry.name,
|
||||
location: (j.normalized_location || j.location || '').trim(),
|
||||
postedAt: toEpochMs(j),
|
||||
});
|
||||
}
|
||||
if (fresh === 0) break; // API ignored offset / looped
|
||||
if (postings.length < PAGE_SIZE) break; // last page
|
||||
}
|
||||
return jobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,127 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Arbeitnow provider — board-wide aggregator feed (EU/DACH-heavy, but
|
||||
// international): https://www.arbeitnow.com/api/job-board-api
|
||||
// Response shape: { data: [ { slug, company_name, title, description, remote,
|
||||
// url, tags, job_types, location, created_at } ], links, meta }
|
||||
//
|
||||
// Jobs are ordered newest-first, 100 per page. The API echoes a rotating
|
||||
// featured `?search=` term into links.next, so page URLs are built directly as
|
||||
// `?page=N` (never by following links.next) to keep the FULL board in view —
|
||||
// scan.mjs's title_filter then gates on the configured titles. Pages are fetched
|
||||
// until one comes back short/empty or the page cap is reached (default 3,
|
||||
// override with `max_pages` on the portal entry).
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: arbeitnow`.
|
||||
|
||||
const FEED_BASE = 'https://www.arbeitnow.com/api/job-board-api';
|
||||
const TRUSTED_HOST = 'www.arbeitnow.com';
|
||||
const PER_PAGE = 100;
|
||||
const DEFAULT_MAX_PAGES = 3;
|
||||
const MAX_PAGES_CAP = 50;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertArbeitnowUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`arbeitnow: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`arbeitnow: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST) {
|
||||
throw new Error(`arbeitnow: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
|
||||
function resolveMaxPages(entry) {
|
||||
const v = entry?.max_pages;
|
||||
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
|
||||
return DEFAULT_MAX_PAGES;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a single Arbeitnow job. Exported for unit tests.
|
||||
*
|
||||
* Field mapping → the normalized Job shape:
|
||||
* - title: `title`, trimmed (items without one are dropped).
|
||||
* - url: `url` — an absolute `https:` posting URL host-locked to
|
||||
* www.arbeitnow.com (an off-host or non-https URL is untrusted and
|
||||
* drops the item). It is the dedup key and is display-only (written
|
||||
* to the pipeline/history, never server-fetched here).
|
||||
* - company: `company_name`, falling back to the portal entry name, then
|
||||
* "Arbeitnow".
|
||||
* - location: `location`, with "Remote" appended when `remote` is true.
|
||||
* - postedAt: `created_at` (epoch SECONDS) → epoch ms (omitted when absent).
|
||||
*
|
||||
* @param {any} j
|
||||
* @param {string} [fallbackCompany]
|
||||
* @returns {{ title: string, url: string, company: string, location: string, postedAt?: number } | null}
|
||||
*/
|
||||
export function normalizeArbeitnowJob(j, fallbackCompany) {
|
||||
if (!j || typeof j !== 'object') return null;
|
||||
|
||||
const title = typeof j.title === 'string' ? j.title.trim() : '';
|
||||
if (!title) return null;
|
||||
|
||||
// url must be an absolute https posting link on www.arbeitnow.com — Arbeitnow
|
||||
// always serves postings there, so an off-host URL is untrusted and dropped.
|
||||
let url = '';
|
||||
const rawUrl = typeof j.url === 'string' ? j.url.trim() : '';
|
||||
if (rawUrl) {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol === 'https:' && parsed.hostname === TRUSTED_HOST) url = parsed.href;
|
||||
} catch {
|
||||
// malformed URL → leave url = '' → dropped below
|
||||
}
|
||||
}
|
||||
if (!url) return null;
|
||||
|
||||
const company =
|
||||
typeof j.company_name === 'string' && j.company_name.trim()
|
||||
? j.company_name.trim()
|
||||
: fallbackCompany || 'Arbeitnow';
|
||||
|
||||
const baseLocation = typeof j.location === 'string' ? j.location.trim() : '';
|
||||
const location = [baseLocation, j.remote === true ? 'Remote' : ''].filter(Boolean).join(', ');
|
||||
|
||||
/** @type {{ title: string, url: string, company: string, location: string, postedAt?: number }} */
|
||||
const job = { title, url, company, location };
|
||||
if (Number.isFinite(j.created_at)) job.postedAt = j.created_at * 1000; // epoch seconds → ms
|
||||
return job;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'arbeitnow',
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
assertArbeitnowUrl(FEED_BASE);
|
||||
const maxPages = resolveMaxPages(entry);
|
||||
const fallbackCompany = entry?.name;
|
||||
const out = [];
|
||||
|
||||
for (let page = 1; page <= maxPages; page++) {
|
||||
// Build the page URL directly (do NOT follow links.next — it carries a
|
||||
// featured `?search=` term that would narrow the board).
|
||||
const url = `${FEED_BASE}?page=${page}`;
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(url, { redirect: 'error' });
|
||||
if (!json || !Array.isArray(json.data)) {
|
||||
throw new Error(
|
||||
`arbeitnow: unexpected API response on page ${page} — expected { data: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`,
|
||||
);
|
||||
}
|
||||
for (const j of json.data) {
|
||||
const normalized = normalizeArbeitnowJob(j, fallbackCompany);
|
||||
if (normalized) out.push(normalized);
|
||||
}
|
||||
if (json.data.length < PER_PAGE) break; // short page → last page reached
|
||||
}
|
||||
return out;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,200 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Arbeitsagentur (Bundesagentur für Arbeit) provider — hits the public Jobsuche
|
||||
// REST API (the same endpoint arbeitsagentur.de uses), so it lives in-process
|
||||
// alongside the other JSON-API providers (greenhouse/ashby shape). One or more
|
||||
// keywords are queried; scan.mjs applies title_filter + location_filter + dedup
|
||||
// afterwards, so this provider over-fetches (recall-first).
|
||||
//
|
||||
// Configure via a `job_boards` (or `tracked_companies`) entry with
|
||||
// `provider: arbeitsagentur` and an `arbeitsagentur:` block:
|
||||
//
|
||||
// - name: Arbeitsagentur — ML/KI Deutschland
|
||||
// provider: arbeitsagentur
|
||||
// arbeitsagentur:
|
||||
// keywords: ["Machine Learning Engineer", "Data Scientist"] # required
|
||||
// wo: Berlin # optional anchor city; omit for nationwide
|
||||
// umkreis: 50 # km radius around `wo` (default 50)
|
||||
// days: 30 # recency window in days (default 30)
|
||||
// size: 100 # results per keyword (1–100, default 100)
|
||||
// remoteNationwide: true # also run a nationwide pass keeping remote-eligible hits
|
||||
// remoteMatch: filter # how that pass detects remote (default 'title'):
|
||||
// # 'filter' — server-side `homeoffice=nv_true` query + pagination; every hit is
|
||||
// # remote-eligible, cheap (no per-job calls). Recommended.
|
||||
// # 'title' — regex on the job title only (cheap; misses body-level remote)
|
||||
// # 'off' — skip the remote pass entirely
|
||||
// remoteMaxPages: 10 # 'filter' mode: max pages to paginate (size each); default 1
|
||||
// enabled: true
|
||||
|
||||
const API_URL = 'https://rest.arbeitsagentur.de/jobboerse/jobsuche-service/pc/v4/jobs';
|
||||
const API_KEY = 'jobboerse-jobsuche'; // public client key the arbeitsagentur.de UI uses
|
||||
const DETAIL_BASE = 'https://www.arbeitsagentur.de/jobsuche/jobdetail/';
|
||||
const REMOTE_RE = /(remote|homeoffice|home[-\s]?office|ortsunabh|deutschlandweit|bundesweit|100\s*%|full[-\s]?remote|fully remote)/i;
|
||||
|
||||
// Clamp a runtime integer into [min, max], falling back to `def` for NaN, so a
|
||||
// stray portals.yml value can't produce empty (size=0) or pathological queries.
|
||||
function intInRange(val, def, min, max) {
|
||||
const n = Number(val);
|
||||
if (!Number.isFinite(n)) return def;
|
||||
return Math.min(max, Math.max(min, Math.trunc(n)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads and sanitizes the entry's `arbeitsagentur:` config block.
|
||||
* @param {{ arbeitsagentur?: any }} entry
|
||||
* @returns {{ keywords: string[], wo: string, umkreis: number, days: number, size: number, remoteNationwide: boolean, remoteMatch: 'title'|'filter'|'off', remoteMaxPages: number }}
|
||||
*/
|
||||
export function parseArbeitsagenturConfig(entry) {
|
||||
const cfg = (entry && entry.arbeitsagentur) || {};
|
||||
const keywords = Array.isArray(cfg.keywords)
|
||||
? cfg.keywords.filter(k => typeof k === 'string' && k.trim()).map(k => k.trim())
|
||||
: [];
|
||||
return {
|
||||
keywords,
|
||||
wo: typeof cfg.wo === 'string' ? cfg.wo.trim() : '',
|
||||
umkreis: intInRange(cfg.umkreis, 50, 0, 1000), // km; only used when `wo` is set
|
||||
days: intInRange(cfg.days, 30, 1, 1000), // recency window
|
||||
size: intInRange(cfg.size, 100, 1, 100), // results per keyword (API max 100)
|
||||
remoteNationwide: cfg.remoteNationwide === true,
|
||||
// Remote-detection mode is config-driven (not hardcoded).
|
||||
remoteMatch: ['title', 'filter', 'off'].includes(cfg.remoteMatch) ? cfg.remoteMatch : 'title',
|
||||
remoteMaxPages: intInRange(cfg.remoteMaxPages, 1, 1, 20),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Assembles a human-readable location from the API's `arbeitsort` object. Most
|
||||
* postings are in Germany; only a non-DE country is appended so the downstream
|
||||
* location_filter can act on it.
|
||||
* @param {any} arbeitsort
|
||||
*/
|
||||
export function buildLocation(arbeitsort) {
|
||||
if (!arbeitsort || typeof arbeitsort !== 'object') return '';
|
||||
const loc = [arbeitsort.ort, arbeitsort.region].filter(Boolean).join(', ');
|
||||
const land = arbeitsort.land;
|
||||
if (land && !/deutschland|germany/i.test(land)) return loc ? `${loc}, ${land}` : land;
|
||||
return loc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes one raw Arbeitsagentur posting into a Job plus its `refnr` (kept
|
||||
* for dedup, stripped before the provider returns). Returns null when the
|
||||
* posting lacks a usable refnr or title.
|
||||
* @param {any} job
|
||||
* @returns {({title: string, url: string, company: string, location: string, refnr: string}) | null}
|
||||
*/
|
||||
export function normalizeJob(job) {
|
||||
const refnr = job && job.refnr;
|
||||
const title = String((job && job.titel) || '').trim();
|
||||
if (!refnr || !title) return null;
|
||||
return {
|
||||
title,
|
||||
url: DETAIL_BASE + encodeURIComponent(String(refnr)),
|
||||
company: String((job && job.arbeitgeber) || '').trim(),
|
||||
location: buildLocation(job && job.arbeitsort),
|
||||
refnr: String(refnr),
|
||||
};
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'arbeitsagentur',
|
||||
|
||||
/**
|
||||
* Fetches and normalizes postings from the Arbeitsagentur Jobsuche API.
|
||||
* @param {{ name?: string, arbeitsagentur?: any }} entry
|
||||
* @param {{ fetchJson: (url: string, opts?: object) => Promise<any> }} ctx
|
||||
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>}
|
||||
*/
|
||||
async fetch(entry, ctx) {
|
||||
const { keywords, wo, umkreis, days, size, remoteNationwide, remoteMatch, remoteMaxPages } = parseArbeitsagenturConfig(entry);
|
||||
if (!keywords.length) {
|
||||
throw new Error(`arbeitsagentur: entry "${entry.name || '(unnamed)'}" has no arbeitsagentur.keywords[]`);
|
||||
}
|
||||
|
||||
/** @param {string} was @param {Record<string,string>} [extra] */
|
||||
const fetchKeyword = async (was, extra = {}) => {
|
||||
const params = new URLSearchParams({
|
||||
was,
|
||||
size: String(size),
|
||||
page: '1',
|
||||
angebotsart: '1', // 1 = ARBEIT (employment; excludes Ausbildung/Selbständigkeit)
|
||||
veroeffentlichtseit: String(days),
|
||||
...extra,
|
||||
});
|
||||
// redirect:'error' prevents SSRF via server-side redirects.
|
||||
const json = await ctx.fetchJson(`${API_URL}?${params.toString()}`, {
|
||||
headers: { 'X-API-Key': API_KEY, accept: 'application/json' },
|
||||
redirect: 'error',
|
||||
timeoutMs: 12_000,
|
||||
});
|
||||
return Array.isArray(json && json.stellenangebote) ? json.stellenangebote : [];
|
||||
};
|
||||
|
||||
const byRef = new Map();
|
||||
const errors = [];
|
||||
let succeeded = 0; // keywords whose primary pass completed (i.e. the source answered)
|
||||
for (const kw of keywords) {
|
||||
let primary;
|
||||
try {
|
||||
// Pass A: commutable radius around `wo`, or a single nationwide pass.
|
||||
primary = wo
|
||||
? await fetchKeyword(kw, { wo, umkreis: String(umkreis) })
|
||||
: await fetchKeyword(kw);
|
||||
succeeded++;
|
||||
} catch (err) {
|
||||
// Recall-first: tolerate a single failed keyword and keep going.
|
||||
errors.push(`"${kw}": ${(err && err.message) || err}`);
|
||||
continue;
|
||||
}
|
||||
// Pass B (optional): a nationwide pass for remote roles hosted at a far HQ
|
||||
// (which the radius pass misses). Detection is config-driven via `remoteMatch`:
|
||||
// 'filter' — server-side `homeoffice=nv_true` query + pagination (every hit is remote)
|
||||
// 'title' — keep only nationwide hits whose title matches the remote regex
|
||||
// Its failure must NOT discard the primary results already fetched above.
|
||||
let wide = [];
|
||||
if (wo && remoteNationwide && remoteMatch !== 'off') {
|
||||
try {
|
||||
if (remoteMatch === 'filter') {
|
||||
// Server-side home-office filter: every hit is remote-eligible, so just
|
||||
// paginate and keep them all — no per-job calls, no title regex.
|
||||
for (let page = 1; page <= remoteMaxPages; page++) {
|
||||
const res = await fetchKeyword(kw, { homeoffice: 'nv_true', page: String(page) });
|
||||
wide.push(...res);
|
||||
if (res.length < size) break; // short page → done
|
||||
}
|
||||
} else { // 'title'
|
||||
const nationwide = await fetchKeyword(kw);
|
||||
wide = nationwide.filter(j => REMOTE_RE.test(String((j && j.titel) || '')));
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push(`"${kw}" (remote pass): ${(err && err.message) || err}`);
|
||||
}
|
||||
}
|
||||
// Pass A (commutable) keeps its city as-is. Pass B roles are remote, so we
|
||||
// append a `Deutschlandweit (Homeoffice)` marker — remote ignores distance,
|
||||
// and this lets scan.mjs's commute-based location_filter pass them via its
|
||||
// always_allow rescue instead of dropping them on the far office city.
|
||||
for (const raw of primary) {
|
||||
const job = normalizeJob(raw);
|
||||
if (job && !byRef.has(job.refnr)) byRef.set(job.refnr, job);
|
||||
}
|
||||
for (const raw of wide) {
|
||||
const job = normalizeJob(raw);
|
||||
if (!job) continue;
|
||||
job.location = job.location ? `${job.location} · Deutschlandweit (Homeoffice)` : 'Deutschlandweit (Homeoffice)';
|
||||
if (!byRef.has(job.refnr)) byRef.set(job.refnr, job);
|
||||
}
|
||||
}
|
||||
|
||||
// Total outage = every primary request failed. A keyword that answered with
|
||||
// zero results is not an outage, so key off the success count, not the
|
||||
// deduped result size — otherwise a legitimately-empty search throws.
|
||||
if (succeeded === 0 && errors.length) {
|
||||
throw new Error(`arbeitsagentur: all ${keywords.length} keyword request(s) failed — ${errors[0]}`);
|
||||
}
|
||||
|
||||
return [...byRef.values()].map(({ refnr, ...job }) => job);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,157 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Ashby provider — hits the public posting-api endpoint.
|
||||
// Auto-detects from careers_url pattern `https://jobs.ashbyhq.com/<slug>`.
|
||||
//
|
||||
// Ashby's public posting-api carries a ~10s+ server-side latency floor
|
||||
// (response time is independent of board size) and rate-limits repeated
|
||||
// unauthenticated hits. The global default timeout (10s, providers/_http.mjs)
|
||||
// sits right on that floor, so requests race the timeout and abort. We give
|
||||
// Ashby a longer timeout plus a backoff+jitter retry (the backoff spaces
|
||||
// requests out to dodge rate-limiting).
|
||||
// See .planning/codebase/ashby-scan-abort-diagnosis.md.
|
||||
const ASHBY_TIMEOUT_MS = 30_000;
|
||||
const ASHBY_RETRIES = 2;
|
||||
|
||||
// Annualization multipliers for different compensation intervals
|
||||
const INTERVAL_MULTIPLIERS = {
|
||||
'1 HOUR': 2080,
|
||||
'1 DAY': 260,
|
||||
'1 WEEK': 52,
|
||||
'2 WEEK': 26,
|
||||
'0.5 MONTH': 24,
|
||||
'1 MONTH': 12,
|
||||
'2 MONTH': 6,
|
||||
'3 MONTH': 4,
|
||||
'6 MONTH': 2,
|
||||
'1 YEAR': 1,
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse compensation data from Ashby job object.
|
||||
* Returns structured salary object with min, max, and currency,
|
||||
* or null if no valid compensation data exists.
|
||||
* @param {any} job - Ashby job object
|
||||
* @returns {{min: number, max: number, currency: string}|null}
|
||||
*/
|
||||
export function parseCompensation(job) {
|
||||
const comp = job?.compensation;
|
||||
if (!comp) return null;
|
||||
|
||||
const interval = /** @type {keyof typeof INTERVAL_MULTIPLIERS} */ (comp.interval || '1 YEAR');
|
||||
const multiplier = INTERVAL_MULTIPLIERS[interval];
|
||||
if (!multiplier) return null;
|
||||
|
||||
// Coerce and validate numeric fields — malformed API payloads must not propagate
|
||||
/** @param {any} v */
|
||||
const normalizeNum = (v) => {
|
||||
if (v == null) return null;
|
||||
if (typeof v === 'string' && v.trim() === '') return null;
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) && n >= 0 ? n : null;
|
||||
};
|
||||
const minValue = normalizeNum(comp.minValue);
|
||||
const maxValue = normalizeNum(comp.maxValue);
|
||||
const currency = typeof comp.currency === 'string' ? comp.currency.trim() : '';
|
||||
|
||||
// If neither min nor max is provided, no valid compensation
|
||||
if (minValue == null && maxValue == null) return null;
|
||||
|
||||
// Annualize the values
|
||||
const min = minValue != null ? minValue * multiplier : null;
|
||||
const max = maxValue != null ? maxValue * multiplier : null;
|
||||
|
||||
// Must have at least one valid annual value
|
||||
if (min == null && max == null) return null;
|
||||
|
||||
// Ensure correct ordering (min <= max)
|
||||
const resolvedMin = /** @type {number} */ (min ?? max);
|
||||
const resolvedMax = /** @type {number} */ (max ?? min);
|
||||
return {
|
||||
min: Math.min(resolvedMin, resolvedMax),
|
||||
max: Math.max(resolvedMin, resolvedMax),
|
||||
currency: currency.toUpperCase(),
|
||||
};
|
||||
}
|
||||
|
||||
/** @param {import('./_types.js').PortalEntry} entry */
|
||||
function resolveApiUrl(entry) {
|
||||
const url = entry.careers_url || '';
|
||||
const match = url.match(/jobs\.ashbyhq\.com\/([^/?#]+)/);
|
||||
if (!match) return null;
|
||||
return `https://api.ashbyhq.com/posting-api/job-board/${match[1]}?includeCompensation=true`;
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
// Build the full location string from primary + secondary locations.
|
||||
// Ashby's posting-api puts extra hiring regions in `secondaryLocations[]`
|
||||
// (each with a region label + a postalAddress). Using only `j.location` drops
|
||||
// them, so an EU-eligible role whose PRIMARY label is e.g. "Canada" reads as
|
||||
// Canada-only and gets wrongly removed by scan.mjs's location_filter. We fold
|
||||
// in each secondary's region, locality, and country so the filter can match
|
||||
// (e.g. "Europe", "Berlin", "Germany"). Deduped, joined with " · ".
|
||||
/** @param {any} j */
|
||||
function formatLocation(j) {
|
||||
const parts = [];
|
||||
if (typeof j.location === 'string' && j.location.trim()) parts.push(j.location.trim());
|
||||
if (Array.isArray(j.secondaryLocations)) {
|
||||
for (const s of j.secondaryLocations) {
|
||||
if (!s || typeof s !== 'object') continue;
|
||||
if (typeof s.location === 'string' && s.location.trim()) parts.push(s.location.trim());
|
||||
const pa = s.address && s.address.postalAddress;
|
||||
if (pa) {
|
||||
for (const k of ['addressLocality', 'addressCountry']) {
|
||||
if (typeof pa[k] === 'string' && pa[k].trim()) parts.push(pa[k].trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...new Set(parts)].join(' · ');
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'ashby',
|
||||
|
||||
detect(entry) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
return apiUrl ? { url: apiUrl } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
if (!apiUrl) throw new Error(`ashby: cannot derive API URL for ${entry.name}`);
|
||||
let lastErr;
|
||||
for (let attempt = 0; attempt <= ASHBY_RETRIES; attempt++) {
|
||||
if (attempt > 0) {
|
||||
// exponential backoff + jitter — spaces out retries to dodge Ashby rate-limiting
|
||||
const backoff = 1000 * 2 ** (attempt - 1) + Math.floor(Math.random() * 500);
|
||||
await sleep(backoff);
|
||||
}
|
||||
try {
|
||||
const json = /** @type {any} */ (await ctx.fetchJson(apiUrl, { timeoutMs: ASHBY_TIMEOUT_MS, redirect: 'error' }));
|
||||
const jobs = Array.isArray(json?.jobs) ? json.jobs : [];
|
||||
return jobs.map(/** @param {any} j */ (j) => ({
|
||||
title: j.title || '',
|
||||
url: j.jobUrl || '',
|
||||
company: entry.name,
|
||||
location: formatLocation(j),
|
||||
salary: parseCompensation(j),
|
||||
postedAt: toEpochMs(j.publishedAt),
|
||||
}));
|
||||
} catch (e) {
|
||||
lastErr = e;
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,224 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Avature provider — parses the public Avature career-site job list.
|
||||
// Auto-detects from a careers_url on `*.avature.net`; a branded custom domain
|
||||
// that proxies Avature needs an explicit `provider: avature` + `api:` pointing
|
||||
// at the Avature origin (or its /careers/SearchJobs URL).
|
||||
//
|
||||
// GET {origin}/careers/SearchJobs?jobOffset=N
|
||||
//
|
||||
// returns a server-rendered page of <article class="article--result"> blocks.
|
||||
// Avature hard-caps the page at 6 results (a jobRecordsPerPage override is
|
||||
// ignored), and paginates via jobOffset in steps of 6 — so this is request-
|
||||
// heavy for large boards; max_pages (default 50 → ~300 postings) bounds it and
|
||||
// can be raised per entry.
|
||||
//
|
||||
// Pagination parameter name is `jobOffset` on classic tenants, but some branded
|
||||
// tenants ignore it and page via a bare `offset` instead (e.g. jobs.siemens.com:
|
||||
// `?jobOffset=6` returns page 1 unchanged, `?offset=6` advances). This is
|
||||
// self-healing: we start with `jobOffset`, and if the first paginated page comes
|
||||
// back identical to page 0 (every id already seen → the param is inert), we
|
||||
// retry that page once with `offset` and adopt whichever key actually advances.
|
||||
// So a new branded tenant needs no configuration. An entry can still pin the key
|
||||
// explicitly via `offset_param` (disables the auto-switch) as an escape hatch.
|
||||
//
|
||||
// Location isn't rendered in every tenant's list (the subtitle is Job ID /
|
||||
// hire-type / posted-date); we extract it when a marker is present and leave it
|
||||
// empty otherwise. postedAt comes from the "Posted DD-Mon-YYYY" subtitle.
|
||||
|
||||
const PAGE_SIZE = 6; // Avature serves exactly 6 results per page
|
||||
const DEFAULT_MAX_PAGES = 50; // ~300 postings; override via entry.max_pages
|
||||
const HARD_MAX_PAGES = 200;
|
||||
// Pause between successive page requests. Avature's 6-results-per-page cap makes
|
||||
// large boards request-heavy (a 999+ board is ~170 pages); firing those with no
|
||||
// gap risks the tenant's WAF rate-limiting the burst. Mirrors workday's
|
||||
// INTER_PAGE_DELAY_MS — only boards that paginate past page 0 pay it.
|
||||
const INTER_PAGE_DELAY_MS = 150;
|
||||
// The bare key we self-heal to when the primary (`jobOffset`) proves inert.
|
||||
const FALLBACK_OFFSET_PARAM = 'offset';
|
||||
|
||||
const MONTHS = { jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5, jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11 };
|
||||
|
||||
/** @param {import('./_types.js').PortalEntry} entry */
|
||||
function resolveConfig(entry) {
|
||||
const raw = entry.api || entry.careers_url || '';
|
||||
let u;
|
||||
try {
|
||||
u = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
|
||||
// Honour an explicit SearchJobs path (branded tenants may prefix a locale,
|
||||
// e.g. /en_US/searchjobs/SearchJobs); otherwise default to the classic path.
|
||||
const searchPath = /\/SearchJobs\b/i.test(u.pathname) ? u.pathname.replace(/\/+$/, '') : '/careers/SearchJobs';
|
||||
return { searchUrl: `${u.origin}${searchPath}`, origin: u.origin };
|
||||
}
|
||||
|
||||
const NAMED = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', '#8226': '•' };
|
||||
/** @param {string} s */
|
||||
function decodeEntities(s) {
|
||||
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z0-9]+);/g, (m, body) => {
|
||||
if (body[0] === '#') {
|
||||
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
||||
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
|
||||
}
|
||||
return NAMED[body.toLowerCase()] ?? m;
|
||||
});
|
||||
}
|
||||
|
||||
/** @param {string} s */
|
||||
function clean(s) {
|
||||
return decodeEntities(s.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
// "Posted 02-May-2026" → epoch ms (UTC midnight). Undefined when absent/unparseable.
|
||||
/** @param {string} block */
|
||||
function parsePosted(block) {
|
||||
const m = block.match(/Posted\s+(\d{1,2})-([A-Za-z]{3})-(\d{4})/);
|
||||
if (!m) return undefined;
|
||||
const mon = MONTHS[m[2].toLowerCase()];
|
||||
if (mon === undefined) return undefined;
|
||||
const ms = Date.UTC(Number(m[3]), mon, Number(m[1]));
|
||||
return Number.isNaN(ms) ? undefined : ms;
|
||||
}
|
||||
|
||||
// Best-effort location: some tenants tag it with a list-item-location span or a
|
||||
// map-marker glyph; most (e.g. Synopsys) render none, so this returns ''.
|
||||
/** @param {string} block */
|
||||
function parseLocation(block) {
|
||||
const m =
|
||||
block.match(/list-item-location[^>]*>([\s\S]*?)<\/span>/i) ||
|
||||
block.match(/class="[^"]*\blocation\b[^"]*"[^>]*>([\s\S]*?)<\/(?:span|div|li)>/i) ||
|
||||
block.match(/glyphicon-map-marker[\s\S]{0,80}?>([^<]{2,60})</i);
|
||||
return m ? clean(m[1]) : '';
|
||||
}
|
||||
|
||||
/** @param {string} htmlText @param {string} origin */
|
||||
export function parseArticles(htmlText, origin) {
|
||||
const out = [];
|
||||
// Tenants vary the result class: Synopsys uses `article--result`, Siemens
|
||||
// appends a position index (`article--result 1`). Accept any suffix.
|
||||
const re = /<article class="article article--result[^"]*"[\s\S]*?<\/article>/g;
|
||||
let a;
|
||||
while ((a = re.exec(htmlText)) !== null) {
|
||||
const block = a[0];
|
||||
// JobDetail path may or may not sit under /careers/ (branded tenants vary),
|
||||
// so anchor on JobDetail/ itself rather than a fixed prefix. Prefer the
|
||||
// `class="link"` title anchor (most tenants); fall back to any JobDetail
|
||||
// anchor for tenants (e.g. Rohde & Schwarz) whose title link carries no
|
||||
// class. Share/mailto buttons url-encode the path (%2FJobDetail%2F) so they
|
||||
// never match the literal `/JobDetail/` and can't be mistaken for the title.
|
||||
const urlM =
|
||||
block.match(/<a[^>]*class="link"[^>]*href="([^"]*\/JobDetail\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/) ||
|
||||
block.match(/<a[^>]*href="([^"]*\/JobDetail\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/);
|
||||
if (!urlM) continue;
|
||||
const title = clean(urlM[2]);
|
||||
if (!title) continue;
|
||||
let url = decodeEntities(urlM[1]);
|
||||
if (!/^https?:\/\//i.test(url)) url = origin + (url.startsWith('/') ? url : '/' + url);
|
||||
const idM = url.match(/\/JobDetail\/[^/]*\/(\d+)/);
|
||||
out.push({
|
||||
id: idM ? idM[1] : url,
|
||||
title,
|
||||
url,
|
||||
location: parseLocation(block),
|
||||
postedAt: parsePosted(block),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'avature',
|
||||
|
||||
detect(entry) {
|
||||
const url = entry.api || entry.careers_url || '';
|
||||
if (typeof url !== 'string') return null;
|
||||
try {
|
||||
const host = new URL(url).host.toLowerCase();
|
||||
if (host === 'avature.net' || host.endsWith('.avature.net')) return { url };
|
||||
} catch {
|
||||
/* not absolute */
|
||||
}
|
||||
return null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const cfg = resolveConfig(entry);
|
||||
if (!cfg) throw new Error(`avature: cannot resolve origin for ${entry.name}`);
|
||||
const maxPages = Math.min(
|
||||
HARD_MAX_PAGES,
|
||||
Number.isFinite(entry.max_pages) && entry.max_pages > 0 ? Number(entry.max_pages) : DEFAULT_MAX_PAGES,
|
||||
);
|
||||
|
||||
// Pagination key. An explicit `offset_param` pins it (and disables the
|
||||
// auto-switch below); otherwise start with `jobOffset` and self-heal. A
|
||||
// non-string/empty override falls back to the default so a malformed entry
|
||||
// can't produce `?=N`.
|
||||
const pinned = typeof entry.offset_param === 'string' && entry.offset_param.trim();
|
||||
let offsetParam = pinned ? entry.offset_param.trim() : 'jobOffset';
|
||||
let canHeal = !pinned; // once the key is pinned, never auto-switch
|
||||
|
||||
const jobs = [];
|
||||
const seen = new Set();
|
||||
const sleep = (ms) => (typeof ctx?.sleep === 'function' ? ctx.sleep(ms) : new Promise((r) => setTimeout(r, ms)));
|
||||
|
||||
const getPage = async (param, page) => {
|
||||
const htmlText = await ctx.fetchText(`${cfg.searchUrl}?${param}=${page * PAGE_SIZE}`, {
|
||||
redirect: 'error',
|
||||
headers: { accept: 'text/html' },
|
||||
});
|
||||
return parseArticles(htmlText, cfg.origin);
|
||||
};
|
||||
// Absorb a page's articles, returning how many were not already seen.
|
||||
const absorb = (articles) => {
|
||||
let fresh = 0;
|
||||
for (const art of articles) {
|
||||
if (seen.has(art.id)) continue;
|
||||
seen.add(art.id);
|
||||
fresh++;
|
||||
jobs.push({
|
||||
title: art.title,
|
||||
url: art.url,
|
||||
company: entry.name,
|
||||
location: art.location,
|
||||
postedAt: art.postedAt,
|
||||
});
|
||||
}
|
||||
return fresh;
|
||||
};
|
||||
|
||||
for (let page = 0; page < maxPages; page++) {
|
||||
if (page > 0) await sleep(INTER_PAGE_DELAY_MS);
|
||||
let articles = await getPage(offsetParam, page);
|
||||
let fresh = absorb(articles);
|
||||
|
||||
// Self-heal: the first paginated page didn't advance — either it repeated
|
||||
// page 0 (all-dup) or came back empty because the primary key is inert
|
||||
// (some tenants echo page 0 for an unknown param, others return nothing).
|
||||
// Retry this page once with the fallback key; if it advances, adopt it for
|
||||
// the remainder. Only on page 1 — inert pagination manifests immediately,
|
||||
// so a later non-advancing page is a genuine end-of-board, not a mismatch.
|
||||
// NOTE: fresh === 0 must be evaluated before the empty-page break below,
|
||||
// or an inert key that returns an empty page 1 would exit without healing.
|
||||
if (fresh === 0 && canHeal && page === 1) {
|
||||
canHeal = false;
|
||||
await sleep(INTER_PAGE_DELAY_MS);
|
||||
const altArticles = await getPage(FALLBACK_OFFSET_PARAM, page);
|
||||
const altFresh = absorb(altArticles);
|
||||
if (altFresh > 0) {
|
||||
offsetParam = FALLBACK_OFFSET_PARAM;
|
||||
articles = altArticles;
|
||||
fresh = altFresh;
|
||||
}
|
||||
}
|
||||
|
||||
if (fresh === 0) break; // empty page / looped / offset ignored / last page
|
||||
if (articles.length < PAGE_SIZE) break; // last page
|
||||
}
|
||||
return jobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,111 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// BambooHR provider — hits the public per-tenant careers list API.
|
||||
// Auto-detects from careers_url pattern `https://<tenant>.bamboohr.com[/...]`.
|
||||
// Per-tenant subdomains are the variable part, so SSRF defence uses a regex
|
||||
// match on `<safe-tenant>.bamboohr.com` rather than a static allowlist
|
||||
// (same approach as the recruitee provider).
|
||||
//
|
||||
// The list endpoint (`/careers/list`) returns lightweight metadata — enough for
|
||||
// the Job contract (title, url, location) at zero token cost. The full JD lives
|
||||
// behind a second `/careers/<id>/detail` request, which the scanner deliberately
|
||||
// skips to stay zero-token (so `description`/`postedAt` are omitted).
|
||||
|
||||
const BAMBOOHR_HOST_RE = /^[a-z0-9][a-z0-9-]*\.bamboohr\.com$/;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertBambooHRUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`bamboohr: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`bamboohr: URL must use HTTPS: ${url}`);
|
||||
if (!BAMBOOHR_HOST_RE.test(parsed.hostname)) {
|
||||
throw new Error(`bamboohr: untrusted hostname "${parsed.hostname}" — must match <tenant>.bamboohr.com`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the tenant origin (`https://<tenant>.bamboohr.com`) from an entry.
|
||||
* Honours an explicit `api:` URL, else parses `careers_url`.
|
||||
* @param {import('./_types.js').PortalEntry} entry
|
||||
* @returns {string | null}
|
||||
*/
|
||||
function resolveOrigin(entry) {
|
||||
const rawApi = typeof entry.api === 'string' ? entry.api : '';
|
||||
const rawCareers = typeof entry.careers_url === 'string' ? entry.careers_url : '';
|
||||
const raw = (rawApi || rawCareers).trim();
|
||||
if (!raw) return null;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (parsed.protocol !== 'https:') return null;
|
||||
if (!BAMBOOHR_HOST_RE.test(parsed.hostname)) return null;
|
||||
return `https://${parsed.hostname}`;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'bamboohr',
|
||||
|
||||
detect(entry) {
|
||||
const origin = resolveOrigin(entry);
|
||||
return origin ? { url: `${origin}/careers/list` } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const origin = resolveOrigin(entry);
|
||||
if (!origin) throw new Error(`bamboohr: cannot derive API URL for ${entry.name}`);
|
||||
const apiUrl = `${origin}/careers/list`;
|
||||
assertBambooHRUrl(apiUrl);
|
||||
// redirect:'error' + the host check above keep the final hostname pinned to
|
||||
// the tenant — a server-side redirect can't bounce us off-domain (SSRF).
|
||||
const json = /** @type {any} */ (await ctx.fetchJson(apiUrl, { redirect: 'error' }));
|
||||
return parseBambooHRResponse(json, entry.name, origin);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a BambooHR `/careers/list` response. Exported for unit tests.
|
||||
*
|
||||
* BambooHR returns:
|
||||
* { meta: {...}, result: [{ id, jobOpeningName,
|
||||
* location: { city?, state? }, isRemote?, employmentStatusLabel? }] }
|
||||
*
|
||||
* - url: built as `<origin>/careers/<id>` — matches the public
|
||||
* `jobOpeningShareUrl`. Rows without a non-empty `id` are dropped (no stable
|
||||
* URL, and url is the scanner's dedup key — a blank id would emit
|
||||
* `/careers/` and collapse distinct postings together).
|
||||
* - location: join `city` + `state`; append "Remote" when `isRemote` is truthy.
|
||||
* BambooHR's `isRemote` is `1`/`true` when set and `null` otherwise.
|
||||
*
|
||||
* @param {any} json
|
||||
* @param {string} companyName
|
||||
* @param {string} origin e.g. "https://acme.bamboohr.com"
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string}>}
|
||||
*/
|
||||
export function parseBambooHRResponse(json, companyName, origin) {
|
||||
const rows = json?.result;
|
||||
if (!Array.isArray(rows)) return [];
|
||||
return rows
|
||||
.filter(j => j && j.jobOpeningName && String(j.id ?? '').trim().length > 0)
|
||||
.map(j => {
|
||||
const loc = j.location || {};
|
||||
const remote = j.isRemote ? 'Remote' : '';
|
||||
const location = [loc.city, loc.state, remote].filter(Boolean).join(', ');
|
||||
const id = String(j.id).trim();
|
||||
return {
|
||||
title: String(j.jobOpeningName),
|
||||
url: `${origin}/careers/${encodeURIComponent(id)}`,
|
||||
company: companyName,
|
||||
location,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// beesite (milch & zucker GJB) provider — the search backend behind branded
|
||||
// portals like jobs.mercedes-benz.com. The SPA calls a public, no-auth JSON
|
||||
// endpoint on the tenant's *.app.beesite.de host:
|
||||
//
|
||||
// GET {origin}/search?data={URL-encoded JSON}
|
||||
// data = {"LanguageCode":"EN",
|
||||
// "SearchParameters":{"FirstItem":1,"CountItem":100,
|
||||
// "Sort":[{"Criterion":"PublicationStartDate","Direction":"DESC"}],
|
||||
// "MatchedObjectDescriptor":[…field list…]},
|
||||
// "SearchCriteria":[…optional facet filters…]}
|
||||
// → {"SearchResult":{"SearchResultCount":N,"SearchResultCountAll":TOTAL,
|
||||
// "SearchResultItems":[{"MatchedObjectId":"200325",
|
||||
// "MatchedObjectDescriptor":{"PositionID":"mer0003yhd",
|
||||
// "PositionTitle":…,"PositionURI":"https://jobs.mercedes-benz.com/…",
|
||||
// "PositionLocation":[{"CityName":"Bremen"}],
|
||||
// "PublicationStartDate":"2026-07-04"}}]}}
|
||||
//
|
||||
// PositionURI is the BRANDED job page (jobs.mercedes-benz.com/…), so listings
|
||||
// link straight to the public posting. FirstItem is 1-based; requesting past
|
||||
// the end returns an empty item list. We sort newest-first so a bounded walk
|
||||
// (max_pages / MAX_JOBS) keeps the most recent postings.
|
||||
//
|
||||
// Facet criteria use tenant-internal numeric term codes (Mercedes: country
|
||||
// 329 = Germany, discoverable via MatchedObjectDescriptor "Facet:…" queries).
|
||||
// A portals.yml entry can pin them via:
|
||||
// beesite:
|
||||
// searchCriteria:
|
||||
// - { CriterionName: PositionLocation.Country, CriterionValue: [329] }
|
||||
|
||||
const PAGE_SIZE = 100; // verified: the endpoint happily serves 100+/page
|
||||
const MAX_PAGES = 40; // safety cap on request count (40*100 = 4000 postings)
|
||||
const MAX_JOBS = 1000; // cap total postings pulled (newest-first sort)
|
||||
const PAGE_DELAY_MS = 150; // polite pacing between page requests
|
||||
|
||||
const DESCRIPTOR = [
|
||||
'PositionID',
|
||||
'PositionTitle',
|
||||
'PositionURI',
|
||||
'PositionLocation.CityName',
|
||||
'PublicationStartDate',
|
||||
];
|
||||
|
||||
/** @param {import('./_types.js').PortalEntry} entry */
|
||||
export function resolveConfig(entry) {
|
||||
const raw = entry.api || entry.careers_url || '';
|
||||
let u;
|
||||
try {
|
||||
u = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
|
||||
const host = u.host.toLowerCase();
|
||||
if (host !== 'beesite.de' && !host.endsWith('.beesite.de')) return null;
|
||||
const cfgBlock = entry.beesite && typeof entry.beesite === 'object' ? entry.beesite : {};
|
||||
return {
|
||||
searchApi: `${u.origin}/search`,
|
||||
languageCode: typeof cfgBlock.languageCode === 'string' ? cfgBlock.languageCode : 'EN',
|
||||
searchCriteria: Array.isArray(cfgBlock.searchCriteria) ? cfgBlock.searchCriteria : [],
|
||||
};
|
||||
}
|
||||
|
||||
/** Build the ?data= payload for one page. @param {any} cfg @param {number} firstItem */
|
||||
export function buildSearchUrl(cfg, firstItem) {
|
||||
const data = {
|
||||
LanguageCode: cfg.languageCode,
|
||||
SearchParameters: {
|
||||
FirstItem: firstItem,
|
||||
CountItem: PAGE_SIZE,
|
||||
Sort: [{ Criterion: 'PublicationStartDate', Direction: 'DESC' }],
|
||||
MatchedObjectDescriptor: DESCRIPTOR,
|
||||
},
|
||||
SearchCriteria: cfg.searchCriteria,
|
||||
};
|
||||
return `${cfg.searchApi}?data=${encodeURIComponent(JSON.stringify(data))}`;
|
||||
}
|
||||
|
||||
// "2026-07-04" → epoch ms (UTC for determinism).
|
||||
/** @param {unknown} raw @returns {number | undefined} */
|
||||
export function parseBeesiteDate(raw) {
|
||||
const m = typeof raw === 'string' ? raw.trim().match(/^(\d{4})-(\d{2})-(\d{2})$/) : null;
|
||||
if (!m) return undefined;
|
||||
const month = Number(m[2]);
|
||||
const day = Number(m[3]);
|
||||
if (month < 1 || month > 12 || day < 1 || day > 31) return undefined;
|
||||
const ms = Date.UTC(Number(m[1]), month - 1, day);
|
||||
return Number.isFinite(ms) ? ms : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map one search response to raw {id, title, url, location, postedAt} records.
|
||||
* @param {any} json
|
||||
* @returns {{total: number|null, rows: Array<{id:string,title:string,url:string,location:string,postedAt?:number}>}}
|
||||
*/
|
||||
export function parseSearchResult(json) {
|
||||
const sr = json?.SearchResult;
|
||||
const total = typeof sr?.SearchResultCountAll === 'number' ? sr.SearchResultCountAll : null;
|
||||
const items = Array.isArray(sr?.SearchResultItems) ? sr.SearchResultItems : [];
|
||||
const rows = [];
|
||||
for (const item of items) {
|
||||
const d = item?.MatchedObjectDescriptor;
|
||||
if (!d) continue;
|
||||
const id = item.MatchedObjectId != null ? String(item.MatchedObjectId) : String(d.PositionID || '');
|
||||
const title = String(d.PositionTitle || '').replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
const url = String(d.PositionURI || '').trim();
|
||||
if (!id || !title || !/^https?:\/\//i.test(url)) continue;
|
||||
const locs = Array.isArray(d.PositionLocation) ? d.PositionLocation : [];
|
||||
const cities = [];
|
||||
for (const l of locs) {
|
||||
const c = String(l?.CityName || '').trim();
|
||||
if (c && !cities.includes(c)) cities.push(c);
|
||||
}
|
||||
rows.push({
|
||||
id,
|
||||
title,
|
||||
url,
|
||||
location: cities.join(' / '),
|
||||
postedAt: parseBeesiteDate(d.PublicationStartDate),
|
||||
});
|
||||
}
|
||||
return { total, rows };
|
||||
}
|
||||
|
||||
/** Resolve the page cap: positive integer `max_pages`, else default. */
|
||||
function resolveMaxPages(entry) {
|
||||
const v = entry?.max_pages;
|
||||
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES);
|
||||
return MAX_PAGES;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'beesite',
|
||||
|
||||
detect(entry) {
|
||||
const url = entry.api || entry.careers_url || '';
|
||||
if (typeof url !== 'string') return null;
|
||||
return resolveConfig({ api: url }) ? { url } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const cfg = resolveConfig(entry);
|
||||
if (!cfg) throw new Error(`beesite: cannot resolve search host for ${entry.name}`);
|
||||
|
||||
const wait = (ms) => (ctx.sleep ? ctx.sleep(ms) : new Promise((r) => setTimeout(r, ms)));
|
||||
const maxPages = resolveMaxPages(entry);
|
||||
const jobs = [];
|
||||
const seen = new Set();
|
||||
let total = null;
|
||||
|
||||
for (let page = 0; page < maxPages; page++) {
|
||||
if (page > 0) await wait(PAGE_DELAY_MS);
|
||||
const json = await ctx.fetchJson(buildSearchUrl(cfg, page * PAGE_SIZE + 1), {
|
||||
redirect: 'error',
|
||||
headers: { accept: 'application/json' },
|
||||
});
|
||||
const { total: pageTotal, rows } = parseSearchResult(json);
|
||||
if (total === null) total = pageTotal;
|
||||
if (rows.length === 0) break; // past the last page
|
||||
|
||||
let fresh = 0;
|
||||
for (const row of rows) {
|
||||
if (seen.has(row.id)) continue;
|
||||
seen.add(row.id);
|
||||
fresh++;
|
||||
const job = { title: row.title, url: row.url, company: entry.name, location: row.location };
|
||||
if (typeof row.postedAt === 'number') job.postedAt = row.postedAt;
|
||||
jobs.push(job);
|
||||
if (jobs.length >= MAX_JOBS) break;
|
||||
}
|
||||
if (fresh === 0) break; // server ignored FirstItem (or we've looped)
|
||||
if (jobs.length >= MAX_JOBS) break;
|
||||
if (total !== null && (page + 1) * PAGE_SIZE >= total) break;
|
||||
}
|
||||
return jobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,136 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Breezy HR provider — hits the public per-tenant board feed.
|
||||
// Auto-detects from careers_url pattern `https://<tenant>.breezy.hr[/...]`.
|
||||
// Per-tenant subdomains are the variable part, so SSRF defence uses a regex
|
||||
// match on `<safe-tenant>.breezy.hr` rather than a static allowlist (same
|
||||
// approach as the recruitee / bamboohr providers).
|
||||
//
|
||||
// Breezy boards expose every published position as a public JSON array at
|
||||
// `<tenant>.breezy.hr/json` — title, absolute url, location, and a published
|
||||
// date, all in the list payload at zero token cost (no per-job request, so the
|
||||
// scanner stays zero-token). Breezy's authenticated REST API (api.breezy.hr) is
|
||||
// intentionally NOT used; only the public board feed.
|
||||
|
||||
const BREEZY_HOST_RE = /^[a-z0-9][a-z0-9-]*\.breezy\.hr$/;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertBreezyUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`breezy: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`breezy: URL must use HTTPS: ${url}`);
|
||||
if (!BREEZY_HOST_RE.test(parsed.hostname)) {
|
||||
throw new Error(`breezy: untrusted hostname "${parsed.hostname}" — must match <tenant>.breezy.hr`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the tenant origin (`https://<tenant>.breezy.hr`) from an entry.
|
||||
* Honours an explicit `api:` URL, else parses `careers_url`.
|
||||
* @param {import('./_types.js').PortalEntry} entry
|
||||
* @returns {string | null}
|
||||
*/
|
||||
function resolveOrigin(entry) {
|
||||
const rawApi = typeof entry.api === 'string' ? entry.api : '';
|
||||
const rawCareers = typeof entry.careers_url === 'string' ? entry.careers_url : '';
|
||||
const raw = (rawApi || rawCareers).trim();
|
||||
if (!raw) return null;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (parsed.protocol !== 'https:') return null;
|
||||
if (!BREEZY_HOST_RE.test(parsed.hostname)) return null;
|
||||
return `https://${parsed.hostname}`;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'breezy',
|
||||
|
||||
detect(entry) {
|
||||
const origin = resolveOrigin(entry);
|
||||
return origin ? { url: `${origin}/json` } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const origin = resolveOrigin(entry);
|
||||
if (!origin) throw new Error(`breezy: cannot derive API URL for ${entry.name}`);
|
||||
const apiUrl = `${origin}/json`;
|
||||
assertBreezyUrl(apiUrl);
|
||||
// redirect:'error' + the host check above keep the final hostname pinned to
|
||||
// the tenant — a server-side redirect can't bounce us off-domain (SSRF).
|
||||
const json = /** @type {any} */ (await ctx.fetchJson(apiUrl, { redirect: 'error' }));
|
||||
return parseBreezyResponse(json, entry.name);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a Breezy `<tenant>.breezy.hr/json` response. Exported for unit tests.
|
||||
*
|
||||
* Breezy returns a top-level array of positions:
|
||||
* [{ name, url, published_date?,
|
||||
* location: { name?, city?, state?, country?: { name }, is_remote? } }]
|
||||
*
|
||||
* - url: Breezy supplies an absolute posting URL on the tenant domain
|
||||
* (`https://<tenant>.breezy.hr/p/<id>-<slug>`); it is the Job contract's dedup
|
||||
* key. The per-offer URL is display-only (recorded in the pipeline/history,
|
||||
* never server-fetched here), so it is not host-locked — only a well-formed
|
||||
* `https:` URL is required; rows without one are dropped.
|
||||
* - location: prefer the ready-made `location.name`; else assemble from
|
||||
* city / state / country.name, appending "Remote" when `is_remote` is truthy.
|
||||
* - postedAt: parsed from the ISO `published_date` when present and valid — Breezy
|
||||
* gives it for free in the list payload, so recency consumers can use it.
|
||||
*
|
||||
* @param {any} json
|
||||
* @param {string} companyName
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parseBreezyResponse(json, companyName) {
|
||||
const rows = Array.isArray(json) ? json : [];
|
||||
const out = [];
|
||||
for (const j of rows) {
|
||||
if (!j || !j.name) continue;
|
||||
|
||||
// Resolve the posting URL (dedup key). Require a well-formed https: URL;
|
||||
// a non-https or malformed/missing URL drops the row.
|
||||
let url = '';
|
||||
const rawUrl = typeof j.url === 'string' ? j.url : '';
|
||||
if (rawUrl) {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol === 'https:') url = parsed.href;
|
||||
} catch { /* malformed → drop */ }
|
||||
}
|
||||
if (!url) continue;
|
||||
|
||||
const loc = j.location || {};
|
||||
const remote = loc.is_remote ? 'Remote' : '';
|
||||
const assembled = [loc.city, loc.state, loc.country?.name].filter(Boolean).join(', ');
|
||||
const base = (typeof loc.name === 'string' && loc.name.trim()) ? loc.name.trim() : assembled;
|
||||
const location = remote && !/remote/i.test(base)
|
||||
? [base, remote].filter(Boolean).join(', ')
|
||||
: base;
|
||||
|
||||
const job = {
|
||||
title: String(j.name),
|
||||
url,
|
||||
company: companyName,
|
||||
location,
|
||||
};
|
||||
|
||||
const ts = Date.parse(j.published_date);
|
||||
if (!Number.isNaN(ts)) job.postedAt = ts;
|
||||
|
||||
out.push(job);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Comeet (Spark Hire Recruit) provider — hits the public, no-auth careers API.
|
||||
//
|
||||
// The positions endpoint needs BOTH a company-uid and a per-tenant token:
|
||||
// https://www.comeet.co/careers-api/2.0/company/<uid>/positions?token=<token>
|
||||
// Neither is derivable from a branded careers URL, so unlike greenhouse/recruitee
|
||||
// there is no slug→API shortcut: the full API URL must be supplied via the
|
||||
// `api:` field (or pasted into `careers_url`). The API host is a single fixed
|
||||
// origin, so the SSRF defence pins hostname to www.comeet.co AND requires the
|
||||
// /careers-api/ path prefix (rather than a per-tenant subdomain regex).
|
||||
|
||||
const COMEET_API_HOST = 'www.comeet.co';
|
||||
|
||||
/** @param {unknown} raw */
|
||||
function isComeetApiUrl(raw) {
|
||||
if (typeof raw !== 'string' || !raw) return false;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return parsed.protocol === 'https:' && parsed.hostname === COMEET_API_HOST && parsed.pathname.startsWith('/careers-api/');
|
||||
}
|
||||
|
||||
/** @param {string} url */
|
||||
function assertComeetUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`comeet: invalid URL: ${redactToken(url)}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`comeet: URL must use HTTPS: ${redactToken(url)}`);
|
||||
if (parsed.hostname !== COMEET_API_HOST)
|
||||
throw new Error(`comeet: untrusted hostname "${parsed.hostname}" — must be ${COMEET_API_HOST}`);
|
||||
if (!parsed.pathname.startsWith('/careers-api/'))
|
||||
throw new Error(`comeet: URL path must be the careers-api endpoint: ${redactToken(url)}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
// Redact the per-tenant ?token= so neither the (informational, possibly-logged)
|
||||
// DetectHit url nor a thrown validation error carries the secret. Best-effort:
|
||||
// falls back to a regex strip when the value can't be parsed as a URL.
|
||||
function redactToken(url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (parsed.searchParams.has('token')) parsed.searchParams.set('token', 'REDACTED');
|
||||
return parsed.href;
|
||||
} catch {
|
||||
return typeof url === 'string' ? url.replace(/([?&]token=)[^&#]*/gi, '$1REDACTED') : url;
|
||||
}
|
||||
}
|
||||
|
||||
/** @param {import('./_types.js').PortalEntry} entry */
|
||||
function resolveApiUrl(entry) {
|
||||
if (isComeetApiUrl(entry.api)) return entry.api;
|
||||
// Fall back to careers_url only when it already IS the full careers-api URL
|
||||
// (the branded www.comeet.com/jobs/... page carries no token, so it can't be
|
||||
// turned into an API call).
|
||||
if (isComeetApiUrl(entry.careers_url)) return entry.careers_url;
|
||||
return null;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'comeet',
|
||||
|
||||
detect(entry) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
// The DetectHit url is informational (the framework may log it), so strip
|
||||
// the secret ?token= before returning it — fetch() re-resolves the real
|
||||
// URL from the entry, so redaction here is safe.
|
||||
return apiUrl ? { url: redactToken(apiUrl) } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
if (!apiUrl) throw new Error(`comeet: cannot derive API URL for ${entry.name} (set api: to the full careers-api positions URL)`);
|
||||
assertComeetUrl(apiUrl);
|
||||
// redirect:'error' prevents SSRF via server-side redirects; combined with
|
||||
// assertComeetUrl above it guarantees the final hostname stays www.comeet.co.
|
||||
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
|
||||
return parseComeetResponse(json, entry.name);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a Comeet careers-api positions response. Exported for unit tests.
|
||||
*
|
||||
* Comeet returns a top-level ARRAY of position objects:
|
||||
* [{ name, location: { name, is_remote }, url_active_page,
|
||||
* url_comeet_hosted_page, time_updated, ... }]
|
||||
*
|
||||
* - url: prefer `url_active_page` (the tenant's live careers page), fall back to
|
||||
* `url_comeet_hosted_page` (the Comeet-hosted page). Both are public, display-
|
||||
* only URLs (recorded in the pipeline/history, never server-fetched here), so
|
||||
* they are NOT host-locked. Require a well-formed https: URL; a position whose
|
||||
* URL is missing/non-https/malformed is dropped (url is the dedup key).
|
||||
* - location: `location.name`, appending "Remote" when `location.is_remote`.
|
||||
*
|
||||
* @param {any} json
|
||||
* @param {string} companyName
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parseComeetResponse(json, companyName) {
|
||||
const positions = Array.isArray(json) ? json : [];
|
||||
return positions
|
||||
.map(row => {
|
||||
// Coerce each row to a safe object so null / non-object members can't throw.
|
||||
const j = (row && typeof row === 'object') ? row : {};
|
||||
// Resolve a display-only https URL; drop the position if none is usable.
|
||||
let url = '';
|
||||
const rawUrl = j.url_active_page || j.url_comeet_hosted_page || '';
|
||||
if (typeof rawUrl === 'string' && rawUrl) {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol === 'https:') url = parsed.href;
|
||||
} catch {
|
||||
// malformed URL → leave url = ''
|
||||
}
|
||||
}
|
||||
|
||||
const loc = j.location || {};
|
||||
const remote = loc.is_remote ? 'Remote' : '';
|
||||
const base = (typeof loc.name === 'string' && loc.name.trim()) ? loc.name.trim() : '';
|
||||
// Append "Remote" only when the base location doesn't already say so.
|
||||
const location = remote && !/remote/i.test(base)
|
||||
? [base, remote].filter(Boolean).join(', ')
|
||||
: base;
|
||||
|
||||
return {
|
||||
title: (typeof j.name === 'string' ? j.name.trim() : ''),
|
||||
url,
|
||||
location,
|
||||
company: companyName,
|
||||
postedAt: toEpochMs(j.time_updated),
|
||||
};
|
||||
})
|
||||
.filter(job => job.title && job.url);
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Dassault Systèmes provider — hits the public Exalead "card search" API that
|
||||
// powers www.3ds.com/careers/jobs. Single-company provider (like ibm.mjs): the
|
||||
// endpoint is global to 3ds.com, so there's no per-tenant config.
|
||||
//
|
||||
// GET https://www.3ds.com/apisearch/card_search_api
|
||||
// ?lang=en
|
||||
// &r=f/card_content_type/career # only career cards
|
||||
// &r=f/card_content_categories_facet/cards language/en # only the English copy
|
||||
// &start={offset} # 0-based, 10 hits/page
|
||||
//
|
||||
// The response is Exalead XML (Content-Type: application/javascript), not JSON.
|
||||
// We don't need a real XML parser — each posting is one <Hit>…</Hit> block whose
|
||||
// fields are <Meta name="…"><MetaString name="value">…</MetaString> pairs.
|
||||
//
|
||||
// Why BOTH refinements matter (learned the hard way):
|
||||
// * The `type=career` URL shorthand returns a *global* career-content index
|
||||
// (~43k hits) where Dassault's own jobs are a small minority ranked first and
|
||||
// the rest is third-party aggregated content (bcit.ca, dejobs.org, …). Using
|
||||
// the proper `f/card_content_type/career` facet instead narrows to Dassault's
|
||||
// own postings (5340 hits, every cta_1 URL on 3ds.com).
|
||||
// * Without the language refinement each job is duplicated across ~12 languages.
|
||||
// `cards language/en` collapses that to the 445 English postings.
|
||||
// As a safety net we still keep only hits whose public URL is on *.3ds.com, so a
|
||||
// future index change can't leak foreign postings back in.
|
||||
//
|
||||
// The Taleo apply backend (talentacquisition.3ds.com) is out of scope — we list
|
||||
// from Exalead and link to the public 3ds.com job page.
|
||||
|
||||
const BASE = 'https://www.3ds.com/apisearch/card_search_api';
|
||||
const REFINES = ['f/card_content_type/career', 'f/card_content_categories_facet/cards language/en'];
|
||||
const PAGE_SIZE = 10; // Exalead returns 10 hits/page
|
||||
const MAX_PAGES = 60; // safety cap on request count (~600 postings; the en set is ~445)
|
||||
const MAX_JOBS = 1000; // cap total postings pulled
|
||||
|
||||
/** @param {number} start */
|
||||
export function buildUrl(start) {
|
||||
const p = new URLSearchParams();
|
||||
p.set('lang', 'en'); // keeps facet labels (Country/City/…) in English
|
||||
for (const r of REFINES) p.append('r', r);
|
||||
p.set('start', String(start));
|
||||
return `${BASE}?${p.toString()}`;
|
||||
}
|
||||
|
||||
// Minimal HTML entity decoder — titles/URLs/locations carry named (&, ')
|
||||
// and numeric (ü / ü) entities. Mirrors successfactors.mjs.
|
||||
const NAMED_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
|
||||
/** @param {string} s */
|
||||
function decodeEntities(s) {
|
||||
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (m, body) => {
|
||||
if (body[0] === '#') {
|
||||
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
||||
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
|
||||
}
|
||||
return NAMED_ENTITIES[body.toLowerCase()] ?? m;
|
||||
});
|
||||
}
|
||||
|
||||
// Pull every <Meta name="X"><MetaString name="value">V</MetaString> pair from one
|
||||
// <Hit> block into a Map. First value per name wins.
|
||||
/** @param {string} hitXml @returns {Map<string,string>} */
|
||||
function metaMap(hitXml) {
|
||||
const map = new Map();
|
||||
const re = /<Meta name="([^"]+)"[^>]*>\s*<MetaString[^>]*name="value"[^>]*>([\s\S]*?)<\/MetaString>/g;
|
||||
let m;
|
||||
while ((m = re.exec(hitXml)) !== null) {
|
||||
if (!map.has(m[1])) map.set(m[1], m[2]);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// content_categories is a flat "Label/Value Label/Value …" string. Values can hold
|
||||
// spaces and commas ("United Kingdom, Cambridge"), so we can't split on whitespace —
|
||||
// we anchor on the known label tokens and slice each value up to the next label.
|
||||
// Labels come back localized even at lang=en in some responses (the API has been
|
||||
// seen emitting Spanish País/Ciudad), so we accept the common locale variants.
|
||||
const LOC_LABELS = [
|
||||
'Category', 'Type', 'Country', 'City', 'Products', 'Year', // en
|
||||
'Categoría', 'Categoria', 'Tipo', 'País', 'Pais', 'Ciudad', 'Productos', 'Año', 'Ano', 'Área', 'Area', // es
|
||||
'Catégorie', 'Pays', 'Ville', 'Produits', 'Année', 'Annee', // fr
|
||||
];
|
||||
const CITY_LABELS = new Set(['City', 'Ciudad', 'Ville']);
|
||||
const COUNTRY_LABELS = new Set(['Country', 'País', 'Pais', 'Pays']);
|
||||
const LABEL_RE = new RegExp('(^|\\s)(' + LOC_LABELS.join('|') + ')\\/', 'g');
|
||||
|
||||
/** @param {string} categories @returns {{city: string, country: string}} */
|
||||
function parseCategories(categories) {
|
||||
const marks = [];
|
||||
let m;
|
||||
LABEL_RE.lastIndex = 0;
|
||||
while ((m = LABEL_RE.exec(categories)) !== null) {
|
||||
marks.push({ label: m[2], keyStart: m.index + m[1].length, valStart: m.index + m[0].length });
|
||||
}
|
||||
let city = '';
|
||||
let country = '';
|
||||
for (let i = 0; i < marks.length; i++) {
|
||||
const end = i + 1 < marks.length ? marks[i + 1].keyStart : categories.length;
|
||||
const value = decodeEntities(categories.slice(marks[i].valStart, end)).trim();
|
||||
if (!city && CITY_LABELS.has(marks[i].label)) city = value;
|
||||
else if (!country && COUNTRY_LABELS.has(marks[i].label)) country = value;
|
||||
}
|
||||
return { city, country };
|
||||
}
|
||||
|
||||
// "2026/07/03 18:22:13" → epoch ms (parsed as UTC so it's deterministic across
|
||||
// machine timezones; the API doesn't state a zone, and consumers only use this
|
||||
// for coarse recency ranking).
|
||||
/** @param {string} ts @returns {number | undefined} */
|
||||
function toEpochMs(ts) {
|
||||
const m = /^(\d{4})\/(\d{2})\/(\d{2})[ T](\d{2}):(\d{2}):(\d{2})/.exec((ts || '').trim());
|
||||
if (!m) return undefined;
|
||||
const [, y, mo, d, h, mi, s] = m.map(Number);
|
||||
const ms = Date.UTC(y, mo - 1, d, h, mi, s);
|
||||
return Number.isFinite(ms) ? ms : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one Exalead card-search XML page into normalized jobs.
|
||||
* Deduped by card_id within the page; the fetch loop dedups again across pages.
|
||||
* Keeps only real Dassault-hosted postings (public URL on *.3ds.com).
|
||||
* @param {string} xml @param {string} entryName
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number, _id: string}>}
|
||||
*/
|
||||
export function parseHits(xml, entryName) {
|
||||
const hits = typeof xml === 'string' ? xml.match(/<Hit\b[\s\S]*?<\/Hit>/g) : null;
|
||||
if (!hits) return [];
|
||||
|
||||
const byId = new Map();
|
||||
for (const hit of hits) {
|
||||
const meta = metaMap(hit);
|
||||
const title = decodeEntities((meta.get('content_title') || '').replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
|
||||
const url = decodeEntities((meta.get('content_cta_1_url') || '').trim());
|
||||
if (!title || !url) continue;
|
||||
|
||||
// Safety net: only accept postings hosted on 3ds.com. The refined query
|
||||
// already guarantees this, but a broadened index must never leak through.
|
||||
let host;
|
||||
try {
|
||||
host = new URL(url).host.toLowerCase();
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (host !== '3ds.com' && !host.endsWith('.3ds.com')) continue;
|
||||
|
||||
const { city, country } = parseCategories(meta.get('content_categories') || '');
|
||||
const postedAt = toEpochMs(meta.get('content_start_datetime') || meta.get('card_update_timestamp') || '');
|
||||
// card_id is the natural dedup key; fall back to the URL when absent.
|
||||
const id = (meta.get('card_id') || '').trim() || url;
|
||||
if (byId.has(id)) continue;
|
||||
|
||||
const job = { title, url, company: entryName || 'Dassault Systèmes', location: city || country, _id: id };
|
||||
if (postedAt !== undefined) job.postedAt = postedAt;
|
||||
byId.set(id, job);
|
||||
}
|
||||
return [...byId.values()];
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'dassault',
|
||||
|
||||
detect(entry) {
|
||||
const url = entry.api || entry.careers_url || '';
|
||||
if (typeof url !== 'string') return null;
|
||||
// Match the host, not a path segment, so evil.com/x.3ds.com can't spoof it.
|
||||
try {
|
||||
const host = new URL(url).host.toLowerCase();
|
||||
if (host === '3ds.com' || host.endsWith('.3ds.com')) return { url };
|
||||
} catch {
|
||||
/* not an absolute URL */
|
||||
}
|
||||
return null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const jobs = [];
|
||||
const seen = new Set();
|
||||
for (let page = 0; page < MAX_PAGES; page++) {
|
||||
const xml = await ctx.fetchText(buildUrl(page * PAGE_SIZE), {
|
||||
redirect: 'error',
|
||||
headers: { accept: 'application/xml, text/xml, */*' },
|
||||
});
|
||||
const parsed = parseHits(xml, entry.name);
|
||||
if (parsed.length === 0) break; // past the last page
|
||||
|
||||
let fresh = 0;
|
||||
for (const job of parsed) {
|
||||
if (seen.has(job._id)) continue;
|
||||
seen.add(job._id);
|
||||
fresh++;
|
||||
const { _id, ...clean } = job; // drop the internal dedup key
|
||||
jobs.push(clean);
|
||||
}
|
||||
if (fresh === 0) break; // server ignored the offset / looped
|
||||
if (jobs.length >= MAX_JOBS) break;
|
||||
}
|
||||
return jobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,134 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Get on Board provider — board-wide feed for the tech "programming" category
|
||||
// (https://www.getonbrd.com/api/v0/categories/programming/jobs). Public,
|
||||
// zero-auth JSON:API. `expand[]=company` embeds the company so its name is
|
||||
// available at the list level. The broad category feed is fetched (not the
|
||||
// server-side ?query= search, which requires a query and narrows results) so
|
||||
// scan.mjs's title_filter can gate on the configured titles instead. Pages are
|
||||
// fetched until one comes back short/empty or the page cap is reached (default
|
||||
// 3, override with `max_pages` on the portal entry).
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: getonbrd`.
|
||||
|
||||
const FEED_BASE = 'https://www.getonbrd.com/api/v0/categories/programming/jobs';
|
||||
const TRUSTED_HOST = 'www.getonbrd.com';
|
||||
const PER_PAGE = 100;
|
||||
const DEFAULT_MAX_PAGES = 3;
|
||||
const MAX_PAGES_CAP = 50;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertGetonbrdUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`getonbrd: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`getonbrd: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST) {
|
||||
throw new Error(`getonbrd: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
|
||||
function resolveMaxPages(entry) {
|
||||
const v = entry?.max_pages;
|
||||
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
|
||||
return DEFAULT_MAX_PAGES;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a single Get on Board job (JSON:API resource). Exported for tests.
|
||||
*
|
||||
* Field mapping → the normalized Job shape:
|
||||
* - title: `attributes.title`, trimmed (items without one are dropped).
|
||||
* - url: `links.public_url` — an absolute `https:` posting URL host-locked
|
||||
* to www.getonbrd.com (off-host or non-https drops the item). It is
|
||||
* the dedup key and is display-only (never server-fetched here).
|
||||
* - company: `attributes.company.data.attributes.name` (from `expand[]=company`),
|
||||
* falling back to the portal entry name, then "Get on Board".
|
||||
* - location: "Remote" when `attributes.remote` is true, else the joined
|
||||
* `attributes.countries` (an array of country names in the live
|
||||
* API; a plain string is tolerated for older payloads).
|
||||
* - postedAt: `attributes.published_at` (epoch SECONDS) → epoch ms (omitted when absent).
|
||||
*
|
||||
* @param {any} j
|
||||
* @param {string} [fallbackCompany]
|
||||
* @returns {{ title: string, url: string, company: string, location: string, postedAt?: number } | null}
|
||||
*/
|
||||
export function normalizeGetonbrdJob(j, fallbackCompany) {
|
||||
if (!j || typeof j !== 'object' || !j.attributes || typeof j.attributes !== 'object') return null;
|
||||
const attr = j.attributes;
|
||||
|
||||
const title = typeof attr.title === 'string' ? attr.title.trim() : '';
|
||||
if (!title) return null;
|
||||
|
||||
// url must be an absolute https posting link on www.getonbrd.com.
|
||||
let url = '';
|
||||
const rawUrl = j.links && typeof j.links.public_url === 'string' ? j.links.public_url.trim() : '';
|
||||
if (rawUrl) {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol === 'https:' && parsed.hostname === TRUSTED_HOST) url = parsed.href;
|
||||
} catch {
|
||||
// malformed URL → leave url = '' → dropped below
|
||||
}
|
||||
}
|
||||
if (!url) return null;
|
||||
|
||||
const name = attr.company?.data?.attributes?.name;
|
||||
const company =
|
||||
typeof name === 'string' && name.trim() ? name.trim() : fallbackCompany || 'Get on Board';
|
||||
|
||||
// `attributes.countries` is an array of country names in the live API (e.g.
|
||||
// ["Chile"]); older/edge payloads may send a plain string. Handle both.
|
||||
let location = '';
|
||||
if (attr.remote === true) {
|
||||
location = 'Remote';
|
||||
} else if (Array.isArray(attr.countries)) {
|
||||
location = attr.countries
|
||||
.filter((c) => typeof c === 'string' && c.trim())
|
||||
.map((c) => c.trim())
|
||||
.join(', ');
|
||||
} else if (typeof attr.countries === 'string') {
|
||||
location = attr.countries.trim();
|
||||
}
|
||||
|
||||
/** @type {{ title: string, url: string, company: string, location: string, postedAt?: number }} */
|
||||
const job = { title, url, company, location };
|
||||
// `attributes.published_at` is epoch SECONDS → convert to ms (omitted when absent).
|
||||
if (Number.isFinite(attr.published_at) && attr.published_at > 0) job.postedAt = attr.published_at * 1000;
|
||||
return job;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'getonbrd',
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
assertGetonbrdUrl(FEED_BASE);
|
||||
const maxPages = resolveMaxPages(entry);
|
||||
const fallbackCompany = entry?.name;
|
||||
const out = [];
|
||||
|
||||
for (let page = 1; page <= maxPages; page++) {
|
||||
const url = `${FEED_BASE}?per_page=${PER_PAGE}&expand[]=company&page=${page}`;
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(url, { redirect: 'error' });
|
||||
if (!json || !Array.isArray(json.data)) {
|
||||
throw new Error(
|
||||
`getonbrd: unexpected API response on page ${page} — expected { data: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`,
|
||||
);
|
||||
}
|
||||
for (const j of json.data) {
|
||||
const normalized = normalizeGetonbrdJob(j, fallbackCompany);
|
||||
if (normalized) out.push(normalized);
|
||||
}
|
||||
if (json.data.length < PER_PAGE) break; // short page → last page reached
|
||||
}
|
||||
return out;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,248 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Glints provider — hits the public GraphQL v2-alc endpoint.
|
||||
//
|
||||
// Glints (glints.com) covers Singapore, Indonesia, Malaysia, and Vietnam.
|
||||
// Their internal API is a no-auth GraphQL endpoint at /api/v2-alc/graphql that
|
||||
// powers the job search page. The schema is reverse-engineered and may change.
|
||||
//
|
||||
// This provider is designed for explicit `provider: glints` in portals.yml.
|
||||
// Auto-detection is not supported — Glints is a job board aggregator, not
|
||||
// a company ATS.
|
||||
//
|
||||
// Portal entry fields (all optional except `provider`):
|
||||
// api — GraphQL endpoint URL (default: https://glints.com/api/v2-alc/graphql)
|
||||
// searchKeywords — Search keywords string (default: '')
|
||||
// countryCode — Two-letter country code (default: "ID" for Indonesia)
|
||||
// pageSize — Results per page (default: 30)
|
||||
// maxPages — Maximum pages (default: 3)
|
||||
// graphqlQuery — Custom GraphQL query string. If not provided, the
|
||||
// built-in default query is used.
|
||||
|
||||
const DEFAULT_API = 'https://glints.com/api/v2-alc/graphql';
|
||||
const DEFAULT_COUNTRY = 'ID';
|
||||
const DEFAULT_PAGE_SIZE = 30;
|
||||
const DEFAULT_MAX_PAGES = 3;
|
||||
|
||||
const ALLOWED_GLINTS_HOSTS = new Set([
|
||||
'glints.com',
|
||||
'www.glints.com',
|
||||
'glints.id',
|
||||
]);
|
||||
|
||||
// Default GraphQL query — reverse-engineered from glints.com/id search.
|
||||
// Uses the searchJobsV3 operation which replaced the older opportunities query.
|
||||
const DEFAULT_GRAPHQL_QUERY = `
|
||||
query searchJobsV3($data: JobSearchConditionInput!) {
|
||||
searchJobsV3(data: $data) {
|
||||
jobsInPage {
|
||||
id
|
||||
title
|
||||
company {
|
||||
name
|
||||
brandName
|
||||
}
|
||||
city {
|
||||
name
|
||||
}
|
||||
country {
|
||||
code
|
||||
name
|
||||
}
|
||||
salaries {
|
||||
salaryType
|
||||
salaryMode
|
||||
maxAmount
|
||||
minAmount
|
||||
CurrencyCode
|
||||
}
|
||||
createdAt
|
||||
}
|
||||
expInfo
|
||||
hasMore
|
||||
}
|
||||
}`;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertGlintsUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`glints: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`glints: URL must use HTTPS: ${url}`);
|
||||
if (!ALLOWED_GLINTS_HOSTS.has(parsed.hostname))
|
||||
throw new Error(`glints: untrusted hostname "${parsed.hostname}" — must be one of: ${[...ALLOWED_GLINTS_HOSTS].join(', ')}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the job detail base URL from the API hostname.
|
||||
* @param {string} apiUrl
|
||||
* @returns {string}
|
||||
*/
|
||||
function deriveBaseUrl(apiUrl) {
|
||||
try {
|
||||
const parsed = new URL(apiUrl);
|
||||
return `${parsed.protocol}//${parsed.hostname}`;
|
||||
} catch {
|
||||
return 'https://glints.com';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single Glints job (searchJobsV3 response) into the canonical Job shape.
|
||||
*
|
||||
* This parser is exported as a named export for unit tests.
|
||||
*
|
||||
* @param {any} item — raw GraphQL result item from searchJobsV3
|
||||
* @param {string} baseUrl — scheme + hostname for resolving relative URLs
|
||||
* @param {string} fallbackCompany — company name fallback from portal entry
|
||||
* @returns {{title: string, url: string, company: string, location: string, postedAt: number|undefined}|null}
|
||||
*/
|
||||
export function parseGlintsItem(item, baseUrl, fallbackCompany) {
|
||||
if (!item || typeof item !== 'object') return null;
|
||||
|
||||
const title = (item.title || '').trim();
|
||||
if (!title) return null;
|
||||
|
||||
// Build job URL from the job ID
|
||||
const jobId = (item.id || '').trim();
|
||||
if (!jobId) return null;
|
||||
const url = `${baseUrl}/id/opportunities/jobs/${jobId}`;
|
||||
|
||||
// Validate URL hostname
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const hostname = parsed.hostname;
|
||||
const allowed = ALLOWED_GLINTS_HOSTS.has(hostname) || hostname.endsWith('.glints.com');
|
||||
if (!allowed) return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const company = (item.company?.name || item.company?.brandName || fallbackCompany || '').trim();
|
||||
const location = (item.city?.name || '').trim();
|
||||
const postedAt = toEpochMs(item.createdAt);
|
||||
|
||||
return { title, url, company, location, ...(postedAt != null ? { postedAt } : {}) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a single GraphQL query page.
|
||||
* @param {string} apiUrl
|
||||
* @param {string} query
|
||||
* @param {object} variables
|
||||
* @param {import('./_types.js').Context} ctx
|
||||
* @returns {Promise<any>}
|
||||
*/
|
||||
async function graphqlPage(apiUrl, query, variables, ctx) {
|
||||
const body = JSON.stringify({ operationName: 'searchJobsV3', query, variables });
|
||||
// Glints firewall blocks non-browser User-Agents, so use a real Chrome UA
|
||||
try {
|
||||
const res = await ctx.fetchJson(apiUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
'origin': 'https://glints.com',
|
||||
'referer': 'https://glints.com/id/opportunities/jobs/explore',
|
||||
},
|
||||
body,
|
||||
redirect: 'error',
|
||||
});
|
||||
return res;
|
||||
} catch (err) {
|
||||
// On POST, some servers return non-JSON errors; attempt text fallback
|
||||
if (err.status && err.body) {
|
||||
let detail = '';
|
||||
try {
|
||||
const parsed = JSON.parse(err.body);
|
||||
detail = parsed.errors?.[0]?.message || err.body.slice(0, 200);
|
||||
} catch {
|
||||
detail = err.body.slice(0, 200);
|
||||
}
|
||||
throw new Error(`glints: HTTP ${err.status} — ${detail}`);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'glints',
|
||||
|
||||
detect(_entry) {
|
||||
// Glints is a job board aggregator, not a company ATS.
|
||||
// Auto-detection is intentionally not supported —
|
||||
// use `provider: glints` explicitly in portals.yml.
|
||||
return null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const apiUrl = entry.api || DEFAULT_API;
|
||||
assertGlintsUrl(apiUrl);
|
||||
const baseUrl = deriveBaseUrl(apiUrl);
|
||||
|
||||
const query = entry.graphqlQuery || DEFAULT_GRAPHQL_QUERY;
|
||||
const keywords = entry.searchKeywords || '';
|
||||
const country = entry.countryCode || DEFAULT_COUNTRY;
|
||||
const pageSize = Number(entry.pageSize) || DEFAULT_PAGE_SIZE;
|
||||
const maxPages = Number(entry.maxPages) || DEFAULT_MAX_PAGES;
|
||||
const fallbackCompany = entry.name || '';
|
||||
|
||||
const allJobs = [];
|
||||
|
||||
for (let page = 1; page <= maxPages; page++) {
|
||||
const variables = {
|
||||
data: {
|
||||
SearchTerm: keywords,
|
||||
CountryCode: country,
|
||||
includeExternalJobs: true,
|
||||
pageSize: pageSize,
|
||||
page: page,
|
||||
},
|
||||
};
|
||||
|
||||
let json;
|
||||
try {
|
||||
json = /** @type {any} */ (await graphqlPage(apiUrl, query, variables, ctx));
|
||||
} catch (err) {
|
||||
if (page === 1) throw err;
|
||||
console.error(`glints: page ${page} fetch failed — ${err.message}`);
|
||||
break;
|
||||
}
|
||||
|
||||
const jobsInPage = json?.data?.searchJobsV3?.jobsInPage;
|
||||
if (!Array.isArray(jobsInPage)) {
|
||||
if (page === 1) throw new Error(`glints: unexpected API response — ${JSON.stringify(json).slice(0, 200)}`);
|
||||
break;
|
||||
}
|
||||
|
||||
if (jobsInPage.length === 0) break;
|
||||
|
||||
for (const item of jobsInPage) {
|
||||
const job = parseGlintsItem(item, baseUrl, fallbackCompany);
|
||||
if (job) allJobs.push(job);
|
||||
}
|
||||
|
||||
// Stop if no more pages
|
||||
if (json?.data?.searchJobsV3?.hasMore === false) break;
|
||||
if (jobsInPage.length < pageSize) break;
|
||||
|
||||
// Rate-limit courtesy delay
|
||||
await new Promise(resolve => setTimeout(resolve, 300));
|
||||
}
|
||||
|
||||
return allJobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,76 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Greenhouse provider — hits the public boards-api JSON endpoint.
|
||||
// Handles both explicit `api:` URLs and auto-detection from `careers_url`.
|
||||
|
||||
const ALLOWED_GREENHOUSE_HOSTS = new Set([
|
||||
'boards-api.greenhouse.io',
|
||||
'boards.greenhouse.io',
|
||||
'job-boards.greenhouse.io',
|
||||
'job-boards.eu.greenhouse.io',
|
||||
]);
|
||||
|
||||
/** @param {string} url */
|
||||
function assertGreenhouseUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`greenhouse: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`greenhouse: URL must use HTTPS: ${url}`);
|
||||
if (!ALLOWED_GREENHOUSE_HOSTS.has(parsed.hostname))
|
||||
throw new Error(`greenhouse: untrusted hostname "${parsed.hostname}" — must be one of: ${[...ALLOWED_GREENHOUSE_HOSTS].join(', ')}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
/** @param {import('./_types.js').PortalEntry} entry */
|
||||
function resolveApiUrl(entry) {
|
||||
if (entry.api) {
|
||||
assertGreenhouseUrl(entry.api);
|
||||
return entry.api;
|
||||
}
|
||||
const url = entry.careers_url || '';
|
||||
const match = url.match(/job-boards(?:\.eu)?\.greenhouse\.io\/([^/?#]+)/);
|
||||
if (match) return `https://boards-api.greenhouse.io/v1/boards/${match[1]}/jobs`;
|
||||
return null;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'greenhouse',
|
||||
|
||||
detect(entry) {
|
||||
try {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
return apiUrl ? { url: apiUrl } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
if (!apiUrl) throw new Error(`greenhouse: cannot derive API URL for ${entry.name}`);
|
||||
assertGreenhouseUrl(apiUrl);
|
||||
// redirect:'error' prevents SSRF via server-side redirects; combined with
|
||||
// assertGreenhouseUrl above it guarantees the final hostname stays in the allowlist.
|
||||
const json = /** @type {any} */ (await ctx.fetchJson(apiUrl, { redirect: 'error' }));
|
||||
const jobs = Array.isArray(json?.jobs) ? json.jobs : [];
|
||||
return jobs.filter(/** @param {any} j */ j => j.absolute_url).map(/** @param {any} j */ j => ({
|
||||
title: j.title || '',
|
||||
url: j.absolute_url,
|
||||
company: entry.name,
|
||||
location: j.location?.name || '',
|
||||
postedAt: toEpochMs(j.first_published),
|
||||
}));
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,182 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Hacker News "Ask HN: Who is hiring?" provider — no auth required.
|
||||
//
|
||||
// Algorithm:
|
||||
// 1. Find the current monthly hiring thread via the Algolia HN search API
|
||||
// (search_by_date, tags=story, query="Ask HN Who is hiring").
|
||||
// 2. Fetch the thread's item from the Algolia items API; top-level `children`
|
||||
// are individual job posts left as top-level comments.
|
||||
// 3. Parse each comment: the first non-empty line is treated as the title/header
|
||||
// (many follow "Company | Role | Location | URL" but this is free-form, so
|
||||
// we extract defensively — a URL is pulled out wherever it appears; the
|
||||
// first line becomes the title; company/location are left empty when the
|
||||
// format doesn't match the pipe-delimited convention).
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: hackernews`.
|
||||
|
||||
const SEARCH_URL =
|
||||
'https://hn.algolia.com/api/v1/search_by_date?tags=story&query=Ask%20HN%20Who%20is%20hiring&hitsPerPage=5';
|
||||
|
||||
/** @param {string} id */
|
||||
function itemUrl(id) {
|
||||
return `https://hn.algolia.com/api/v1/items/${id}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan raw text for the first absolute http/https URL. Returns '' if none found.
|
||||
* @param {string} text
|
||||
*/
|
||||
function extractUrl(text) {
|
||||
const m = text.match(/https?:\/\/[^\s<>"')]+/);
|
||||
return m ? m[0].replace(/[.,;!?)]+$/, '') : '';
|
||||
}
|
||||
|
||||
// Named HTML entities we decode in comment bodies. Kept in single map
|
||||
/** @type {Record<string, string>} */
|
||||
const ENTITY_MAP = {
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'"': '"',
|
||||
''': "'",
|
||||
''': "'",
|
||||
' ': ' ',
|
||||
};
|
||||
const ENTITY_RE = /&|<|>|"|'|'| /g;
|
||||
|
||||
/**
|
||||
* Parse a single HN comment text into a normalized job object.
|
||||
* The canonical format is "Company | Role | Location | URL" on the first line,
|
||||
* but posts are free-form — we only guarantee title (first line) and url (first
|
||||
* URL found anywhere in the text). company and location are extracted from the
|
||||
* pipe-delimited header when present; left empty otherwise.
|
||||
*
|
||||
* Exported for unit testing.
|
||||
*
|
||||
* @param {string} text Raw comment text (may contain HTML; tags are stripped).
|
||||
* @param {string} threadUrl Fallback url (the HN thread) if no URL in comment.
|
||||
* @returns {{ title: string, url: string, company: string, location: string } | null}
|
||||
* null when the comment is empty, deleted, or carries no usable title.
|
||||
*/
|
||||
export function parseHnComment(text, threadUrl = '') {
|
||||
if (!text || typeof text !== 'string') return null;
|
||||
|
||||
// Strip HTML. Anchors: keep the href value in place so URL extraction works.
|
||||
// Block-level tags (<p>, <br>, <div>, <li>, headings) become newlines so that
|
||||
// body paragraphs never bleed into the first header line after the join.
|
||||
const plain = text
|
||||
.replace(/<a\s[^>]*href="([^"]+)"[^>]*>.*?<\/a>/gi, (_, href) => href)
|
||||
.replace(/<\/?(?:p|br|div|li|h[1-6])\b[^>]*>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(ENTITY_RE, (m) => ENTITY_MAP[m])
|
||||
.replace(/\r\n/g, '\n')
|
||||
.replace(/\r/g, '\n');
|
||||
|
||||
// Split into lines; first non-blank line is the header.
|
||||
const lines = plain.split('\n').map(l => l.trim()).filter(Boolean);
|
||||
if (lines.length === 0) return null;
|
||||
|
||||
const firstLine = lines[0];
|
||||
if (!firstLine) return null;
|
||||
|
||||
// Try to parse pipe-delimited header: Company | Role | Location | URL (4+ parts)
|
||||
// or Company | Role | Location (3 parts).
|
||||
const parts = firstLine.split('|').map(p => p.trim());
|
||||
|
||||
let company = '';
|
||||
let location = '';
|
||||
|
||||
if (parts.length >= 3) {
|
||||
company = parts[0];
|
||||
// Role is parts[1] — used in title construction below, not stored separately.
|
||||
// Strip any embedded URL from the location field (e.g. when the URL is run
|
||||
// into the same pipe segment rather than given its own 4th segment).
|
||||
location = parts[2].replace(/https?:\/\/\S+/g, '').trim();
|
||||
} else if (parts.length === 2) {
|
||||
company = parts[0];
|
||||
}
|
||||
// Single-part first line: title only, company/location stay empty.
|
||||
|
||||
// Build a clean title from the first line (strip any embedded URL from it).
|
||||
const title = firstLine.replace(/https?:\/\/[^\s]+/g, '').replace(/\s{2,}/g, ' ').trim();
|
||||
if (!title) return null;
|
||||
|
||||
// Find first URL anywhere in the full text.
|
||||
const url = extractUrl(plain) || threadUrl;
|
||||
if (!url) return null;
|
||||
|
||||
return { title, url, company, location };
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the objectID of the latest "Ask HN: Who is hiring?" story.
|
||||
* Returns null when the search yields no matching hits.
|
||||
* @param {unknown} data Parsed Algolia search response.
|
||||
*/
|
||||
export function resolveLatestThreadId(data) {
|
||||
if (!data || typeof data !== 'object') return null;
|
||||
const hits = /** @type {any} */ (data).hits;
|
||||
if (!Array.isArray(hits) || hits.length === 0) return null;
|
||||
|
||||
// Algolia search_by_date returns newest first; find the first hit whose title
|
||||
// matches the monthly thread pattern (case-insensitive).
|
||||
const RE = /ask\s+hn[:\s]+who\s+is\s+hiring/i;
|
||||
for (const hit of hits) {
|
||||
if (hit && typeof hit.objectID === 'string' && typeof hit.title === 'string') {
|
||||
if (RE.test(hit.title)) return hit.objectID;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'hackernews',
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
// Step 1: Find the latest "Who is hiring?" story id.
|
||||
const searchData = await ctx.fetchJson(SEARCH_URL, { redirect: 'error' });
|
||||
const threadId = resolveLatestThreadId(searchData);
|
||||
if (!threadId) {
|
||||
throw new Error('hackernews: could not find "Ask HN: Who is hiring?" thread in search results');
|
||||
}
|
||||
|
||||
const threadHnUrl = `https://news.ycombinator.com/item?id=${threadId}`;
|
||||
|
||||
// Step 2: Fetch the thread item (children = top-level job comments).
|
||||
const item = await ctx.fetchJson(itemUrl(threadId), { redirect: 'error' });
|
||||
if (!item || typeof item !== 'object') {
|
||||
throw new Error(`hackernews: unexpected item response for thread ${threadId}`);
|
||||
}
|
||||
|
||||
const children = /** @type {any} */ (item).children;
|
||||
if (!Array.isArray(children)) return [];
|
||||
|
||||
// Step 3: Parse each comment.
|
||||
const jobs = [];
|
||||
for (const child of children) {
|
||||
// Skip deleted / dead / empty comments.
|
||||
if (!child || child.deleted || child.dead) continue;
|
||||
const text = typeof child.text === 'string' ? child.text : '';
|
||||
if (!text.trim()) continue;
|
||||
|
||||
const parsed = parseHnComment(text, threadHnUrl);
|
||||
if (!parsed) continue;
|
||||
|
||||
jobs.push({
|
||||
title: parsed.title,
|
||||
url: parsed.url,
|
||||
company: parsed.company || (entry.name || 'HN Hiring'),
|
||||
location: parsed.location,
|
||||
// Algolia returns created_at as ISO string.
|
||||
...(child.created_at
|
||||
? { postedAt: Date.parse(child.created_at) || undefined }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
return jobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,152 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// HigherEdJobs.com RSS category feed provider.
|
||||
// (https://www.higheredjobs.com/rss/categoryFeed.cfm?catID=68). The feed is
|
||||
// public, no-auth, and XML, so it is parsed in-process with the same tiny tag
|
||||
// extractor approach as providers/weworkremotely.mjs rather than adding an
|
||||
// XML dependency.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: higheredjobs`.
|
||||
|
||||
const DEFAULT_CAT_ID = 68; // Higher Education category
|
||||
const TRUSTED_HOST = 'www.higheredjobs.com';
|
||||
|
||||
function feedUrlFor(catId) {
|
||||
const catID = typeof catId === 'number' && Number.isFinite(catId) ? catId : DEFAULT_CAT_ID;
|
||||
return `https://www.higheredjobs.com/rss/categoryFeed.cfm?catID=${catID}`;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
function fallbackCompany(entry) {
|
||||
return typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : 'HigherEdJobs';
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'higheredjobs',
|
||||
|
||||
detect(entry) {
|
||||
if (entry?.provider !== 'higheredjobs') return null;
|
||||
return { url: feedUrlFor(entry.cat_id) };
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const feedUrl = feedUrlFor(entry?.cat_id);
|
||||
// redirect:'error' prevents SSRF via server-side redirects; combined with
|
||||
// cleanUrl below it keeps the request pinned to www.higheredjobs.com.
|
||||
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
|
||||
return parseHigherEdJobsFeed(text, fallbackCompany(entry));
|
||||
},
|
||||
};
|
||||
|
||||
function fromCodePoint(cp) {
|
||||
try {
|
||||
return String.fromCodePoint(cp);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// Decode the XML entities that appear in RSS text: numeric (& / ')
|
||||
// and the named five. Numeric forms are decoded first; & is decoded LAST
|
||||
// so a literal "&lt;" yields "<" rather than over-decoding to "<".
|
||||
function decodeXmlEntities(s) {
|
||||
return s
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
|
||||
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
|
||||
function extractText(inner) {
|
||||
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
|
||||
if (cdata) return cdata[1].trim();
|
||||
return decodeXmlEntities(inner).trim();
|
||||
}
|
||||
|
||||
// Extract the text of the first <tag>...</tag> in a block. Returns '' when absent.
|
||||
function tagText(block, tag) {
|
||||
const m = block.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, 'i'));
|
||||
return m ? extractText(m[1]) : '';
|
||||
}
|
||||
|
||||
function cleanUrl(value) {
|
||||
if (!value) return '';
|
||||
const trimmed = value.trim();
|
||||
try {
|
||||
const parsed = new URL(trimmed);
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
// Exact-match host (no subdomain wildcard): higheredjobs serves only
|
||||
// www.higheredjobs.com for posting URLs.
|
||||
const trusted = host === TRUSTED_HOST;
|
||||
return parsed.protocol === 'https:' && trusted ? parsed.href : '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// HigherEdJobs <description> is "Institution Name (City, ST)". Split on the
|
||||
// last " (" to separate company from location. When no parens are present the
|
||||
// whole description is treated as the company and location is empty.
|
||||
function splitDescription(rawDescription, defaultCompany) {
|
||||
const text = rawDescription.trim();
|
||||
const open = text.lastIndexOf(' (');
|
||||
if (open > 0) {
|
||||
const close = text.lastIndexOf(')');
|
||||
const company = text.slice(0, open).trim();
|
||||
const location = close > open ? text.slice(open + 2, close).trim() : text.slice(open + 2).trim();
|
||||
if (company) return { company, location };
|
||||
}
|
||||
return { company: text || defaultCompany, location: '' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse HigherEdJobs' public RSS category feed. Exported for unit tests.
|
||||
*
|
||||
* Shape: `<rss><channel><item>...</item>...</channel></rss>`, each item
|
||||
* carrying `<title>` (plain job title), `<description>` ("Institution (City,
|
||||
* ST)"), `<link>`, `<pubDate>`, and `<guid>`. Company and location are
|
||||
* derived from <description>. The RSS <link> is the dedup key; items without
|
||||
* a usable absolute URL on www.higheredjobs.com are dropped.
|
||||
*
|
||||
* @param {string} xml - raw RSS feed body
|
||||
* @param {string} [defaultCompany] - fallback company for empty descriptions
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parseHigherEdJobsFeed(xml, defaultCompany = 'HigherEdJobs') {
|
||||
if (typeof xml !== 'string') return [];
|
||||
const fallback = typeof defaultCompany === 'string' && defaultCompany.trim() ? defaultCompany.trim() : 'HigherEdJobs';
|
||||
const jobs = [];
|
||||
const blocks = xml.match(/<item\b[^>]*>[\s\S]*?<\/item>/gi) || [];
|
||||
|
||||
for (const item of blocks) {
|
||||
const url = cleanUrl(tagText(item, 'link'));
|
||||
if (!url) continue;
|
||||
|
||||
const title = tagText(item, 'title');
|
||||
if (!title) continue;
|
||||
|
||||
const { company, location } = splitDescription(tagText(item, 'description'), fallback);
|
||||
|
||||
jobs.push({
|
||||
title,
|
||||
company,
|
||||
location,
|
||||
url,
|
||||
postedAt: toEpochMs(tagText(item, 'pubDate')),
|
||||
});
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Himalayas provider - board-wide remote jobs API
|
||||
// (https://himalayas.app/jobs/api?limit=50). Returns { jobs: [...] }. The
|
||||
// full feed is fetched so scan.mjs's title_filter / location_filter can do
|
||||
// the local gating consistently with other zero-token board providers.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: himalayas`.
|
||||
|
||||
const FEED_URL = 'https://himalayas.app/jobs/api?limit=50';
|
||||
const TRUSTED_HOST = 'himalayas.app';
|
||||
|
||||
/** @param {string} url */
|
||||
function assertHimalayasUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`himalayas: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`himalayas: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST) {
|
||||
throw new Error(`himalayas: untrusted hostname "${parsed.hostname}" - must be ${TRUSTED_HOST}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
function cleanText(value) {
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
function cleanHimalayasUrl(value) {
|
||||
const raw = cleanText(value);
|
||||
if (!raw) return '';
|
||||
try {
|
||||
const parsed = new URL(raw);
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
const trusted = host === TRUSTED_HOST || host.endsWith(`.${TRUSTED_HOST}`);
|
||||
return parsed.protocol === 'https:' && trusted ? parsed.href : '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function locationText(value) {
|
||||
if (!Array.isArray(value)) return '';
|
||||
return value
|
||||
.filter(v => typeof v === 'string' && v.trim())
|
||||
.map(v => v.trim())
|
||||
.join(', ');
|
||||
}
|
||||
|
||||
// Himalayas pubDate is currently epoch seconds. Accept milliseconds and
|
||||
// parseable date strings too so the parser survives small API shape changes.
|
||||
function toEpochMs(value) {
|
||||
if (typeof value === 'number' && Number.isFinite(value)) {
|
||||
return value < 1_000_000_000_000 ? value * 1000 : value;
|
||||
}
|
||||
if (typeof value === 'string' && value.trim()) {
|
||||
const numeric = Number(value);
|
||||
if (Number.isFinite(numeric)) return numeric < 1_000_000_000_000 ? numeric * 1000 : numeric;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'himalayas',
|
||||
|
||||
detect(entry) {
|
||||
return entry?.provider === 'himalayas' ? { url: FEED_URL } : null;
|
||||
},
|
||||
|
||||
/**
|
||||
* Fetches and normalizes postings from the Himalayas public feed.
|
||||
* @param {{ provider?: string }} entry - The job_boards entry being processed.
|
||||
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
|
||||
* @returns {Promise<Array<{title: string, url: string, company: string, location: string, postedAt?: number}>>}
|
||||
*/
|
||||
async fetch(entry, ctx) {
|
||||
const feedUrl = assertHimalayasUrl(FEED_URL);
|
||||
// redirect:'error' prevents SSRF via server-side redirects; combined with
|
||||
// assertHimalayasUrl above it keeps the request pinned to himalayas.app.
|
||||
const json = await ctx.fetchJson(feedUrl, { redirect: 'error' });
|
||||
if (!json || !Array.isArray(json.jobs)) {
|
||||
throw new Error(`himalayas: unexpected API response - expected { jobs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
|
||||
}
|
||||
return parseHimalayasResponse(json);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse Himalayas' public jobs API response. Exported for unit tests.
|
||||
*
|
||||
* Shape: `{ jobs: [...] }`, where each job currently carries `title`,
|
||||
* `companyName`, `locationRestrictions`, `applicationLink`, `guid`,
|
||||
* `pubDate`, and `companySlug`. `applicationLink` is preferred over `guid`
|
||||
* and used as the dedup key after HTTPS + host validation.
|
||||
*
|
||||
* @param {unknown} json - raw parsed API response
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parseHimalayasResponse(json) {
|
||||
if (!json || typeof json !== 'object' || !Array.isArray(json.jobs)) return [];
|
||||
|
||||
const jobs = [];
|
||||
for (const item of json.jobs) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
|
||||
const title = cleanText(item.title);
|
||||
if (!title) continue;
|
||||
|
||||
const url = cleanHimalayasUrl(item.applicationLink) || cleanHimalayasUrl(item.guid);
|
||||
if (!url) continue;
|
||||
|
||||
jobs.push({
|
||||
title,
|
||||
url,
|
||||
company: cleanText(item.companyName),
|
||||
location: locationText(item.locationRestrictions),
|
||||
postedAt: toEpochMs(item.pubDate),
|
||||
});
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// IBM careers provider — POSTs to IBM's careers search API (Elasticsearch-style;
|
||||
// the same endpoint the ibm.com/careers/search UI calls). One endpoint serves
|
||||
// every locale, so results are language-agnostic (lang: "zz").
|
||||
//
|
||||
// Configure via a `job_boards` (or `tracked_companies`) entry with `provider: ibm`
|
||||
// and an optional `ibm:` block of facet filters:
|
||||
//
|
||||
// - name: IBM Germany — SWE & Data
|
||||
// provider: ibm
|
||||
// ibm:
|
||||
// country: Germany # field_keyword_05
|
||||
// categories: ["Software Engineering", "Data & Analytics"] # field_keyword_08
|
||||
// enabled: true
|
||||
//
|
||||
// Omit `country` / `categories` to widen the search. Paginates via `from` until
|
||||
// the result set is exhausted or MAX_RECORDS is reached.
|
||||
|
||||
const API_URL = 'https://www-api.ibm.com/search/api/v2';
|
||||
const PAGE_SIZE = 30;
|
||||
const MAX_RECORDS = 600; // safety cap on pagination
|
||||
|
||||
/**
|
||||
* Builds the Elasticsearch post_filter from the entry's `ibm:` config.
|
||||
* @param {{ country?: string, categories?: string[] }} cfg
|
||||
*/
|
||||
export function buildPostFilter(cfg) {
|
||||
const must = [];
|
||||
// Sanitize operator config: keep only non-empty trimmed strings so a stray/
|
||||
// mistyped entry can't inject empty or non-string filter terms.
|
||||
const categories = Array.isArray(cfg.categories)
|
||||
? cfg.categories.filter(c => typeof c === 'string' && c.trim()).map(c => c.trim())
|
||||
: [];
|
||||
if (categories.length) {
|
||||
must.push({ bool: { should: categories.map(c => ({ term: { field_keyword_08: c } })) } });
|
||||
}
|
||||
const country = typeof cfg.country === 'string' ? cfg.country.trim() : '';
|
||||
if (country) must.push({ term: { field_keyword_05: country } });
|
||||
return { bool: { must } };
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes one page of the IBM careers API response into job entries.
|
||||
* Throws if the response doesn't carry the expected `hits.hits[]` shape, so a
|
||||
* silent endpoint change surfaces as a hard error instead of empty results.
|
||||
* @param {any} json - A single API response page.
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string}>}
|
||||
*/
|
||||
export function parseIbmResponse(json) {
|
||||
const hits = json && json.hits && Array.isArray(json.hits.hits) ? json.hits.hits : null;
|
||||
if (!hits) {
|
||||
throw new Error(`ibm: unexpected API response — expected hits.hits[], got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
|
||||
}
|
||||
|
||||
const out = [];
|
||||
for (const h of hits) {
|
||||
const s = (h && h._source) || {};
|
||||
if (typeof s.title !== 'string' || s.title.trim() === '') continue;
|
||||
if (typeof s.url !== 'string' || !/^https?:\/\//i.test(s.url.trim())) continue;
|
||||
const loc = typeof s.field_keyword_19 === 'string' ? s.field_keyword_19.trim() : '';
|
||||
const mode = typeof s.field_keyword_17 === 'string' ? s.field_keyword_17.trim() : '';
|
||||
out.push({
|
||||
title: s.title.trim(),
|
||||
url: s.url.trim(),
|
||||
company: 'IBM',
|
||||
location: [loc, mode].filter(Boolean).join(' · '),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'ibm',
|
||||
|
||||
/**
|
||||
* Fetches and normalizes postings from IBM's careers search API.
|
||||
* @param {{ name?: string, ibm?: { country?: string, categories?: string[] } }} entry
|
||||
* @param {{ fetchJson: (url: string, opts?: object) => Promise<any> }} ctx
|
||||
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>}
|
||||
*/
|
||||
async fetch(entry, ctx) {
|
||||
const postFilter = buildPostFilter(entry.ibm || {});
|
||||
/** @type {Array<{title: string, url: string, company: string, location: string}>} */
|
||||
const out = [];
|
||||
|
||||
for (let from = 0; from < MAX_RECORDS; from += PAGE_SIZE) {
|
||||
const body = {
|
||||
appId: 'careers',
|
||||
scopes: ['careers2'],
|
||||
query: { bool: { must: [] } },
|
||||
post_filter: postFilter,
|
||||
size: PAGE_SIZE,
|
||||
from,
|
||||
sort: [{ _score: 'desc' }, { pageviews: 'desc' }],
|
||||
lang: 'zz',
|
||||
localeSelector: {},
|
||||
sm: { query: '', lang: 'zz' },
|
||||
_source: ['_id', 'title', 'url', 'description', 'language',
|
||||
'field_keyword_05', 'field_keyword_08', 'field_keyword_17',
|
||||
'field_keyword_18', 'field_keyword_19'],
|
||||
};
|
||||
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(API_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', accept: 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
redirect: 'error',
|
||||
});
|
||||
|
||||
out.push(...parseIbmResponse(json));
|
||||
|
||||
// Stop on the first short page. IBM's `total` is unreliable (often
|
||||
// capped at the page size), so don't trust it for the loop bound —
|
||||
// MAX_RECORDS is the hard ceiling.
|
||||
if (json.hits.hits.length < PAGE_SIZE) break;
|
||||
}
|
||||
|
||||
return out;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,142 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// JibeApply provider — hits the /api/jobs endpoint on the same hostname.
|
||||
// Auto-detects from careers_url pattern `https://<slug>.jibeapply.com`.
|
||||
// iCIMS acquired Jibe in 2019; some tenants run it on a branded custom
|
||||
// domain instead. The JSON API at /api/jobs is stable across all tenants.
|
||||
|
||||
// Safety cap on pagination — applied regardless of what the upstream reports
|
||||
// as totalCount, so a misbehaving/compromised API can't drive this into
|
||||
// fetching thousands of pages. Every known real tenant (10/page) stays well
|
||||
// under the default of 50 pages (500 jobs); override with `max_pages` on the
|
||||
// portal entry for a tenant that genuinely exceeds it — each page is a
|
||||
// sequential round-trip, so raising the default for everyone isn't free.
|
||||
const DEFAULT_MAX_PAGES = 50;
|
||||
const MAX_PAGES_CAP = 500;
|
||||
|
||||
// Fallback page size when a response carries no jobs and no usable `count`
|
||||
// (e.g. an empty first page). Confirmed live against real tenants — JibeApply
|
||||
// reports this as `filter.displayLimit` and it has been 10 on every tenant
|
||||
// observed so far.
|
||||
const DEFAULT_PAGE_SIZE = 10;
|
||||
|
||||
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
|
||||
function resolveMaxPages(entry) {
|
||||
const v = entry?.max_pages;
|
||||
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
|
||||
return DEFAULT_MAX_PAGES;
|
||||
}
|
||||
|
||||
function toApiUrl(rawUrl) {
|
||||
let u;
|
||||
try { u = new URL(rawUrl); } catch { return null; }
|
||||
if (u.protocol !== 'https:') return null;
|
||||
if (!/^[a-z0-9-]+\.jibeapply\.com$/i.test(u.hostname)) return null;
|
||||
if (!u.pathname.startsWith('/api/')) {
|
||||
u.pathname = '/api' + (u.pathname.startsWith('/') ? u.pathname : '/' + u.pathname);
|
||||
}
|
||||
return u.href;
|
||||
}
|
||||
|
||||
// Validate an explicit entry.api URL (any HTTPS hostname — used for iCIMS-hosted,
|
||||
// branded JibeApply sites that share the same JSON schema).
|
||||
function validateExplicitApi(apiUrl) {
|
||||
let u;
|
||||
try { u = new URL(apiUrl); } catch { return null; }
|
||||
return u.protocol === 'https:' ? u.href : null;
|
||||
}
|
||||
|
||||
export function parseJibeapplyResponse(json, entry) {
|
||||
let origin = '';
|
||||
try { origin = new URL(entry.careers_url || '').origin; } catch { /* ignore */ }
|
||||
// careers_url isn't required to be a well-formed absolute URL for fetch()
|
||||
// to succeed (an explicit entry.api bypasses it entirely — see toApiUrl
|
||||
// callers below) — fall back to api's origin so job URLs stay absolute
|
||||
// instead of silently degrading to a relative "/jobs/<slug>" path.
|
||||
if (!origin) {
|
||||
try { origin = new URL(entry.api || '').origin; } catch { /* ignore */ }
|
||||
}
|
||||
const items = Array.isArray(json?.jobs) ? json.jobs : [];
|
||||
return items
|
||||
.map(item => {
|
||||
if (item == null) return null;
|
||||
const d = item.data ?? item;
|
||||
const title = String(d.title || '').trim();
|
||||
const slug = d.slug || d.req_id;
|
||||
if (!title || !slug) return null;
|
||||
return {
|
||||
title,
|
||||
url: `${origin}/jobs/${encodeURIComponent(slug)}`,
|
||||
company: String(d.hiring_organization || entry.name || '').trim(),
|
||||
location: d.full_location || [d.city, d.country].filter(Boolean).join(', '),
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'jibeapply',
|
||||
|
||||
detect(entry) {
|
||||
const url = entry.careers_url;
|
||||
if (typeof url !== 'string') return null;
|
||||
const apiUrl = toApiUrl(url);
|
||||
if (!apiUrl) return null;
|
||||
return { url: apiUrl };
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const url = entry.careers_url;
|
||||
if (typeof url !== 'string' || !url) throw new Error('jibeapply: careers_url required');
|
||||
|
||||
// Prefer an explicit entry.api (allows iCIMS-hosted, branded JibeApply sites
|
||||
// that share the same JSON schema but aren't on jibeapply.com).
|
||||
const apiUrl = (typeof entry.api === 'string' && validateExplicitApi(entry.api))
|
||||
|| toApiUrl(url);
|
||||
if (!apiUrl) throw new Error(`jibeapply: cannot derive API URL for ${entry.name}`);
|
||||
const first = await ctx.fetchJson(apiUrl, { redirect: 'error' });
|
||||
const total = first.totalCount ?? 0;
|
||||
// Use the actual number of items returned as page size — some implementations
|
||||
// set `count` to the total rather than the per-page count.
|
||||
const pageSize = first.jobs?.length || first.count || DEFAULT_PAGE_SIZE;
|
||||
const allJobs = [...(first.jobs ?? [])];
|
||||
|
||||
if (total > pageSize && pageSize > 0) {
|
||||
const maxPages = resolveMaxPages(entry);
|
||||
const pages = Math.min(Math.ceil(total / pageSize), maxPages);
|
||||
// Sequential, not concurrent (mirrors providers/4dayweek.mjs, thehub.mjs,
|
||||
// arbeitnow.mjs, workday.mjs) — a single tenant's API has no reason to
|
||||
// receive a burst of parallel requests, and a mid-run failure stops
|
||||
// cleanly with whatever pages were already gathered instead of
|
||||
// discarding them (Promise.all would fail the whole batch on one error).
|
||||
for (let page = 2; page <= pages; page++) {
|
||||
const u2 = new URL(apiUrl);
|
||||
u2.searchParams.set('page', String(page));
|
||||
let json;
|
||||
try {
|
||||
json = await ctx.fetchJson(u2.toString(), { redirect: 'error' });
|
||||
} catch (err) {
|
||||
console.error(`⚠️ jibeapply: ${entry.name} page ${page} fetch failed — ${err.message} (returning ${allJobs.length} jobs fetched so far)`);
|
||||
break;
|
||||
}
|
||||
allJobs.push(...(json.jobs ?? []));
|
||||
}
|
||||
|
||||
// The cap is silent by design (it's a safety net, not a working limit),
|
||||
// but a tenant that actually exceeds it needs to be surfaced —
|
||||
// otherwise the user has no way to notice postings are missing from
|
||||
// their scan.
|
||||
if (Math.ceil(total / pageSize) > maxPages) {
|
||||
console.error(
|
||||
`⚠️ jibeapply: ${entry.name} has more postings than max_pages allows ` +
|
||||
`(fetched ${allJobs.length} of ${total}) — ` +
|
||||
`set max_pages on this portal entry to raise the cap (current: ${maxPages})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return parseJibeapplyResponse({ jobs: allJobs }, entry);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,84 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Jobicy provider — board-wide remote-jobs aggregator feed
|
||||
// (https://jobicy.com/api/v2/remote-jobs?count=50). Returns { jobs: [...] }.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: jobicy`.
|
||||
|
||||
const FEED_URL = 'https://jobicy.com/api/v2/remote-jobs?count=50';
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'jobicy',
|
||||
|
||||
detect(entry) {
|
||||
return entry?.provider === 'jobicy' ? { url: FEED_URL } : null;
|
||||
},
|
||||
|
||||
/**
|
||||
* Fetches and normalizes postings from the Jobicy public feed.
|
||||
* @param {{ name?: string }} entry - The job_boards entry being processed.
|
||||
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
|
||||
* @returns {Promise<Array<{title: string, url: string, company: string, location: string, postedAt?: number}>>}
|
||||
*/
|
||||
async fetch(entry, ctx) {
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(FEED_URL, { redirect: 'error' });
|
||||
if (!json || !Array.isArray(json.jobs)) {
|
||||
throw new Error(`jobicy: unexpected API response — expected { jobs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
|
||||
}
|
||||
|
||||
return parseJobicyResponse(json, entry.name || 'Jobicy');
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a Jobicy API response. Exported for unit tests.
|
||||
*
|
||||
* @param {any} json - Raw response payload.
|
||||
* @param {string} defaultCompany - Fallback company name.
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string}>}
|
||||
*/
|
||||
export function parseJobicyResponse(json, defaultCompany = 'Jobicy') {
|
||||
if (!json || !Array.isArray(json.jobs)) return [];
|
||||
|
||||
const toEpochMs = (value) => {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
};
|
||||
|
||||
return json.jobs
|
||||
.map(j => {
|
||||
if (!j || typeof j !== 'object') return null;
|
||||
|
||||
const title = typeof j.jobTitle === 'string' ? j.jobTitle.trim() : '';
|
||||
if (!title) return null;
|
||||
|
||||
const rawUrl = typeof j.url === 'string' ? j.url.trim() : '';
|
||||
let url = null;
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol === 'https:' && (parsed.hostname === 'jobicy.com' || parsed.hostname === 'www.jobicy.com')) {
|
||||
url = parsed.href;
|
||||
}
|
||||
} catch {
|
||||
// Invalid or malformed URL
|
||||
}
|
||||
if (!url) return null;
|
||||
|
||||
const company = typeof j.companyName === 'string' && j.companyName.trim() ? j.companyName.trim() : defaultCompany;
|
||||
const location = typeof j.jobGeo === 'string' ? j.jobGeo.trim() : '';
|
||||
const postedAt = toEpochMs(j.pubDate);
|
||||
|
||||
return {
|
||||
title,
|
||||
url,
|
||||
company,
|
||||
location,
|
||||
postedAt,
|
||||
};
|
||||
})
|
||||
.filter(j => j !== null);
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Jobspresso provider - public WordPress jobs feed
|
||||
// (https://jobspresso.co/?feed=job_feed). The feed is public, no-auth,
|
||||
// and XML, so it is parsed in-process with the same tiny tag extractor
|
||||
// approach as providers/personio.mjs rather than adding an XML dependency.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: jobspresso`.
|
||||
|
||||
const FEED_URL = 'https://jobspresso.co/?feed=job_feed';
|
||||
const TRUSTED_HOST = 'jobspresso.co';
|
||||
|
||||
/** @param {string} url */
|
||||
function assertJobspressoUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`jobspresso: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== "https:")
|
||||
throw new Error(`jobspresso: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST) {
|
||||
throw new Error(
|
||||
`jobspresso: untrusted hostname "${parsed.hostname}" - must be ${TRUSTED_HOST}`,
|
||||
);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: "jobspresso",
|
||||
|
||||
detect(entry) {
|
||||
return entry?.provider === "jobspresso" ? { url: FEED_URL } : null;
|
||||
},
|
||||
|
||||
async fetch(_entry, ctx) {
|
||||
const feedUrl = assertJobspressoUrl(FEED_URL);
|
||||
// redirect:'error' prevents SSRF via server-side redirects; combined with
|
||||
// assertJobspressoUrl above it keeps the request pinned to jobspresso.co.
|
||||
const text = await ctx.fetchText(feedUrl, { redirect: "error" });
|
||||
return parseJobspressoFeed(text);
|
||||
},
|
||||
};
|
||||
|
||||
function fromCodePoint(cp) {
|
||||
try {
|
||||
return String.fromCodePoint(cp);
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
// Decode the XML entities that appear in RSS text: numeric (& / ')
|
||||
// and the named five. Numeric forms are decoded first; & is decoded LAST
|
||||
// so a literal "&lt;" yields "<" rather than over-decoding to "<".
|
||||
function decodeXmlEntities(s) {
|
||||
return s
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
|
||||
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, "&");
|
||||
}
|
||||
|
||||
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
|
||||
function extractText(inner) {
|
||||
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
|
||||
if (cdata) return cdata[1].trim();
|
||||
return decodeXmlEntities(inner).trim();
|
||||
}
|
||||
|
||||
// Extract the text of the first <tag>...</tag> in a block. Returns '' when absent.
|
||||
function tagText(block, tag) {
|
||||
const m = block.match(
|
||||
new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, "i"),
|
||||
);
|
||||
return m ? extractText(m[1]) : "";
|
||||
}
|
||||
|
||||
function cleanUrl(value) {
|
||||
if (!value) return "";
|
||||
const trimmed = value.trim();
|
||||
try {
|
||||
const parsed = new URL(trimmed);
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
const trusted = host === TRUSTED_HOST || host.endsWith(`.${TRUSTED_HOST}`);
|
||||
return parsed.protocol === "https:" && trusted ? parsed.href : "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Jobspresso's public WordPress jobs feed.
|
||||
*
|
||||
* Shape:
|
||||
* <rss><channel><item>...</item></channel></rss>
|
||||
*
|
||||
* Each item contains:
|
||||
* - title
|
||||
* - link
|
||||
* - pubDate
|
||||
* - job_listing:company
|
||||
* - job_listing:location
|
||||
*
|
||||
* The RSS link is used as the dedup key.
|
||||
* @param {string} xml - raw RSS feed body
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parseJobspressoFeed(xml) {
|
||||
if (typeof xml !== "string") return [];
|
||||
const jobs = [];
|
||||
const blocks = xml.match(/<item\b[^>]*>[\s\S]*?<\/item>/gi) || [];
|
||||
|
||||
for (const item of blocks) {
|
||||
const url = cleanUrl(tagText(item, "link"));
|
||||
if (!url) continue;
|
||||
|
||||
const title = tagText(item, "title");
|
||||
if (!title) continue;
|
||||
const company = tagText(item, "job_listing:company") || "";
|
||||
const location = tagText(item, "job_listing:location");
|
||||
|
||||
jobs.push({
|
||||
title,
|
||||
company,
|
||||
location,
|
||||
url,
|
||||
postedAt: toEpochMs(tagText(item, "pubDate")),
|
||||
});
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Jobstreet / SEEK provider — hits the public SEEK v5 JobSearch REST API.
|
||||
//
|
||||
// Jobstreet (jobstreet.com, jobstreet.co.id, etc.) and SEEK (seek.com.au,
|
||||
// seek.co.nz) share the same SEEK infrastructure. The old chalice-search
|
||||
// v4 API (/api/chalice-search/v4/search) was deprecated; the v5 API at
|
||||
// /api/jobsearch/v5/search is the current replacement.
|
||||
//
|
||||
// This provider is designed for explicit `provider: jobstreet` in portals.yml.
|
||||
// Auto-detection from careers_url is not supported because Jobstreet is a
|
||||
// job board aggregator, not a company ATS.
|
||||
//
|
||||
// Portal entry fields (all optional except `provider`):
|
||||
// api — v5 search endpoint URL (default: https://id.jobstreet.com/api/jobsearch/v5/search)
|
||||
// siteKey — SEEK site key for regional filtering (default: "ID-Main")
|
||||
// searchKeywords — Search keywords, space-separated (default: "")
|
||||
// searchLocation — Location filter (default: "")
|
||||
// pageSize — Results per page (default: 30)
|
||||
// maxPages — Maximum pages to fetch (default: 3)
|
||||
//
|
||||
// Site keys by market:
|
||||
// ID-Main → id.jobstreet.com (Indonesia)
|
||||
// SG-Main → sg.jobstreet.com (Singapore)
|
||||
// MY-Main → my.jobstreet.com (Malaysia)
|
||||
|
||||
const DEFAULT_API = 'https://id.jobstreet.com/api/jobsearch/v5/search';
|
||||
const DEFAULT_SITE_KEY = 'ID-Main';
|
||||
const DEFAULT_PAGE_SIZE = 30;
|
||||
const DEFAULT_MAX_PAGES = 3;
|
||||
|
||||
const ALLOWED_JOBSTREET_HOSTS = new Set([
|
||||
'id.jobstreet.com',
|
||||
'www.jobstreet.com',
|
||||
'www.jobstreet.co.id',
|
||||
'jobstreet.com',
|
||||
'jobstreet.co.id',
|
||||
'sg.jobstreet.com',
|
||||
'my.jobstreet.com',
|
||||
'www.seek.com.au',
|
||||
'www.seek.co.nz',
|
||||
]);
|
||||
|
||||
// v5 API paths (the client-side JS on jobstreet uses these relative paths
|
||||
// resolved against the current origin). We keep the allowlist for SSRF
|
||||
// protection on the base URL, then build the v5 search path from it.
|
||||
const V5_SEARCH_PATH = '/api/jobsearch/v5/search';
|
||||
|
||||
/** @param {string} url */
|
||||
function assertJobstreetUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`jobstreet: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`jobstreet: URL must use HTTPS: ${url}`);
|
||||
if (!ALLOWED_JOBSTREET_HOSTS.has(parsed.hostname))
|
||||
throw new Error(`jobstreet: untrusted hostname "${parsed.hostname}" — must be one of: ${[...ALLOWED_JOBSTREET_HOSTS].join(', ')}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the origin from the API hostname.
|
||||
* e.g. id.jobstreet.com → https://id.jobstreet.com
|
||||
* @param {string} apiUrl
|
||||
* @returns {string}
|
||||
*/
|
||||
function deriveOrigin(apiUrl) {
|
||||
try {
|
||||
const parsed = new URL(apiUrl);
|
||||
return `${parsed.protocol}//${parsed.hostname}`;
|
||||
} catch {
|
||||
return 'https://id.jobstreet.com';
|
||||
}
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single Jobstreet/SEEK v5 search API result into the canonical Job shape.
|
||||
*
|
||||
* The v5 search API returns objects shaped like:
|
||||
* {
|
||||
* id: "92996157",
|
||||
* title: "Facility Engineer",
|
||||
* advertiser: { id: "60960115", description: "PT YOFC International Indonesia" },
|
||||
* companyName: "YOFC International",
|
||||
* locations: [{ label: "Karawang, West Java", countryCode: "ID", ... }],
|
||||
* listingDate: "2026-06-29T02:53:00Z",
|
||||
* listingDateDisplay: "16h ago",
|
||||
* roleId: "facilities-engineer",
|
||||
* salaryLabel: "",
|
||||
* teaser: "...",
|
||||
* workTypes: ["Full time"],
|
||||
* workArrangements: { data: [{ id: "1", label: { text: "On-site" } }] },
|
||||
* ...
|
||||
* }
|
||||
*
|
||||
* This parser is exported as a named export for unit tests.
|
||||
*
|
||||
* @param {any} item — raw v5 API result item
|
||||
* @param {string} origin — scheme + hostname for building job detail URLs
|
||||
* @param {string} fallbackCompany — company name fallback from the portal entry
|
||||
* @returns {{title: string, url: string, company: string, location: string, postedAt: number|undefined}|null}
|
||||
*/
|
||||
export function parseJobstreetItem(item, origin, fallbackCompany) {
|
||||
if (!item || typeof item !== 'object') return null;
|
||||
|
||||
const title = (item.title || '').trim();
|
||||
if (!title) return null;
|
||||
|
||||
// Build job URL from the job ID
|
||||
const jobId = (item.id || '').trim();
|
||||
if (!jobId) return null;
|
||||
const url = `${origin}/id/job/${jobId}`;
|
||||
|
||||
// Validate URL hostname belongs to allowed set
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (!ALLOWED_JOBSTREET_HOSTS.has(parsed.hostname)) return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Prefer advertiser.description for the branded company name, fall back
|
||||
// to companyName (which can be shorter/less specific), then entry name.
|
||||
const company = (item.advertiser?.description || item.companyName || fallbackCompany || '').trim();
|
||||
const location = (item.locations?.[0]?.label || '').trim();
|
||||
const postedAt = toEpochMs(item.listingDate);
|
||||
|
||||
return { title, url, company, location, ...(postedAt != null ? { postedAt } : {}) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the v5 search URL with query parameters.
|
||||
* @param {string} origin — scheme + hostname
|
||||
* @param {object} params
|
||||
* @returns {string}
|
||||
*/
|
||||
function buildSearchUrl(origin, params) {
|
||||
const url = new URL(V5_SEARCH_PATH, origin);
|
||||
const { siteKey, keywords, location, pageSize, page } = params;
|
||||
if (siteKey) url.searchParams.set('siteKey', siteKey);
|
||||
if (keywords) url.searchParams.set('keywords', keywords);
|
||||
if (location) url.searchParams.set('where', location);
|
||||
url.searchParams.set('pageSize', String(pageSize || DEFAULT_PAGE_SIZE));
|
||||
url.searchParams.set('page', String(page || 1));
|
||||
return url.href;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'jobstreet',
|
||||
|
||||
detect(_entry) {
|
||||
// Jobstreet is a job board aggregator, not a company ATS.
|
||||
// Auto-detection from careers_url is intentionally not supported —
|
||||
// use `provider: jobstreet` explicitly in portals.yml.
|
||||
return null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const apiUrl = entry.api || DEFAULT_API;
|
||||
assertJobstreetUrl(apiUrl);
|
||||
const origin = deriveOrigin(apiUrl);
|
||||
|
||||
const siteKey = entry.siteKey || DEFAULT_SITE_KEY;
|
||||
const keywords = entry.searchKeywords || '';
|
||||
const searchLocation = entry.searchLocation || '';
|
||||
const pageSize = Number(entry.pageSize) || DEFAULT_PAGE_SIZE;
|
||||
const maxPages = Number(entry.maxPages) || DEFAULT_MAX_PAGES;
|
||||
const fallbackCompany = entry.name || '';
|
||||
|
||||
const allJobs = [];
|
||||
|
||||
for (let page = 1; page <= maxPages; page++) {
|
||||
const searchUrl = buildSearchUrl(origin, {
|
||||
siteKey,
|
||||
keywords,
|
||||
location: searchLocation,
|
||||
pageSize,
|
||||
page,
|
||||
});
|
||||
|
||||
let json;
|
||||
try {
|
||||
json = /** @type {any} */ (await ctx.fetchJson(searchUrl, { redirect: 'error' }));
|
||||
} catch (err) {
|
||||
// If page 1 fails, surface the error. Later pages failing is non-fatal
|
||||
// — we return whatever we've collected so far.
|
||||
if (page === 1) throw err;
|
||||
console.error(`jobstreet: page ${page} fetch failed — ${err.message}`);
|
||||
break;
|
||||
}
|
||||
|
||||
const data = Array.isArray(json?.data) ? json.data : [];
|
||||
if (data.length === 0) break;
|
||||
|
||||
for (const item of data) {
|
||||
const job = parseJobstreetItem(item, origin, fallbackCompany);
|
||||
if (job) allJobs.push(job);
|
||||
}
|
||||
|
||||
// Stop if we got fewer results than pageSize (last page)
|
||||
if (data.length < pageSize) break;
|
||||
|
||||
// Respect rate limits — small delay between pages
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
}
|
||||
|
||||
return allJobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,128 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// JustJoin.it provider — hits the current candidate offers API.
|
||||
// Browser URLs under https://justjoin.it/job-offers/... are accepted for
|
||||
// detection, but fetches use https://justjoin.it/api/candidate-api/offers.
|
||||
|
||||
const ALLOWED_HOSTS = new Set(['justjoin.it']);
|
||||
const API_BASE = 'https://justjoin.it/api/candidate-api/offers';
|
||||
const JOB_BASE = 'https://justjoin.it/job-offer/';
|
||||
const PAGE_SIZE = 100;
|
||||
const MAX_PAGES = 50;
|
||||
|
||||
function assertJustJoinUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`justjoin: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`justjoin: URL must use HTTPS: ${url}`);
|
||||
if (!ALLOWED_HOSTS.has(parsed.hostname)) {
|
||||
throw new Error(`justjoin: untrusted hostname "${parsed.hostname}" — must be justjoin.it`);
|
||||
}
|
||||
if (!parsed.pathname.startsWith('/job-offers') && parsed.pathname !== '/api/candidate-api/offers') {
|
||||
throw new Error(`justjoin: URL path must be /job-offers or /api/candidate-api/offers: ${url}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function detectUrl(entry) {
|
||||
const url = entry.api || entry.careers_url || '';
|
||||
if (typeof url !== 'string' || !url.trim()) return null;
|
||||
try {
|
||||
const parsed = assertJustJoinUrl(url);
|
||||
return { url: parsed.href };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeLocation(offer) {
|
||||
const parts = [];
|
||||
const workplace = String(offer?.workplaceType || '').trim();
|
||||
if (workplace) parts.push(workplace);
|
||||
if (Array.isArray(offer?.locations)) {
|
||||
for (const location of offer.locations) {
|
||||
const city = String(location?.city || '').trim();
|
||||
if (city) parts.push(city);
|
||||
}
|
||||
}
|
||||
const city = String(offer?.city || '').trim();
|
||||
if (city) parts.push(city);
|
||||
return [...new Set(parts.filter(Boolean))].join(', ');
|
||||
}
|
||||
|
||||
function postedAtMillis(value) {
|
||||
const time = Date.parse(String(value || ''));
|
||||
return Number.isFinite(time) ? time : undefined;
|
||||
}
|
||||
|
||||
function buildApiUrl(entry, from) {
|
||||
const apiUrl = entry.api || API_BASE;
|
||||
const parsed = assertJustJoinUrl(apiUrl);
|
||||
if (parsed.pathname !== '/api/candidate-api/offers') {
|
||||
parsed.pathname = '/api/candidate-api/offers';
|
||||
parsed.search = '';
|
||||
}
|
||||
parsed.searchParams.set('from', String(from));
|
||||
parsed.searchParams.set('itemsCount', String(Number(entry.page_size || PAGE_SIZE)));
|
||||
parsed.searchParams.set('cityRadius', String(Number(entry.city_radius || 30)));
|
||||
parsed.searchParams.set('currency', String(entry.currency || 'pln').toLowerCase());
|
||||
parsed.searchParams.set('orderBy', 'descending');
|
||||
parsed.searchParams.set('sortBy', 'publishedAt');
|
||||
parsed.searchParams.set('keywordType', 'any');
|
||||
parsed.searchParams.set('isPromoted', 'true');
|
||||
return parsed.href;
|
||||
}
|
||||
|
||||
export function parseJustJoinResponse(json) {
|
||||
if (!json || !Array.isArray(json.data)) {
|
||||
throw new Error(`justjoin: unexpected API response — expected { data: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
|
||||
}
|
||||
return json.data
|
||||
.filter(offer => offer && typeof offer === 'object')
|
||||
.map(offer => {
|
||||
const slug = String(offer.slug || '').trim();
|
||||
const title = String(offer.title || '').trim();
|
||||
if (!slug || !title) return null;
|
||||
return {
|
||||
title,
|
||||
url: `${JOB_BASE}${slug}`,
|
||||
company: String(offer.companyName || '').trim(),
|
||||
location: normalizeLocation(offer),
|
||||
postedAt: postedAtMillis(offer.publishedAt),
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'justjoin',
|
||||
|
||||
detect(entry) {
|
||||
return detectUrl(entry);
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
if (!detectUrl(entry)) throw new Error('justjoin: careers_url or api must be a trusted justjoin.it URL');
|
||||
|
||||
const jobs = [];
|
||||
let from = 0;
|
||||
const maxPages = Number(entry.max_pages || MAX_PAGES);
|
||||
for (let page = 0; page < maxPages; page++) {
|
||||
const url = buildApiUrl(entry, from);
|
||||
const json = await ctx.fetchJson(url, { redirect: 'error' });
|
||||
const parsed = parseJustJoinResponse(json);
|
||||
jobs.push(...parsed);
|
||||
|
||||
const next = json?.meta?.next?.cursor;
|
||||
if (next == null || parsed.length === 0) break;
|
||||
from = Number(next);
|
||||
if (!Number.isFinite(from)) break;
|
||||
}
|
||||
return jobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,132 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Landing.jobs provider — board-wide aggregator feed (tech, Europe-focused):
|
||||
// https://landing.jobs/api/v1/jobs
|
||||
// Response shape: a JSON ARRAY of
|
||||
// { id, title, url, locations: [{ city, country_code }], remote, published_at,
|
||||
// created_at, type, tags, gross_salary_low, gross_salary_high, ... }
|
||||
//
|
||||
// NOTE: the v1 feed carries no company-name field — the employer slug only
|
||||
// appears in the posting URL path (`https://landing.jobs/at/<slug>/<job>`), so
|
||||
// `company` is derived best-effort from that slug (humanized) and falls back to
|
||||
// the portal entry name. The whole active set is returned in one call.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: landingjobs`.
|
||||
|
||||
const FEED_URL = 'https://landing.jobs/api/v1/jobs';
|
||||
const TRUSTED_HOST = 'landing.jobs';
|
||||
|
||||
/** @param {string} url */
|
||||
function assertLandingUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`landingjobs: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`landingjobs: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST) {
|
||||
throw new Error(`landingjobs: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse.
|
||||
function toEpochMs(value) {
|
||||
if (typeof value !== 'string' || !value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a best-effort company name from a Landing.jobs posting URL.
|
||||
* Posting URLs are `https://landing.jobs/at/<slug>/<job>`; the `<slug>` is
|
||||
* humanized (hyphens/underscores → spaces, title-cased). Returns '' when the
|
||||
* URL is not the expected `/at/<slug>/…` shape. Exported for unit tests.
|
||||
* @param {string} url
|
||||
*/
|
||||
export function companyFromUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
const segs = parsed.pathname.split('/').filter(Boolean);
|
||||
if (segs[0] !== 'at' || !segs[1]) return '';
|
||||
return segs[1]
|
||||
.replace(/[-_]+/g, ' ')
|
||||
.trim()
|
||||
.replace(/\b\w/g, c => c.toUpperCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a single Landing.jobs job. Exported for unit tests.
|
||||
*
|
||||
* Field mapping → the normalized Job shape:
|
||||
* - title: `title`, trimmed (postings without one are dropped).
|
||||
* - url: `url` — host-locked to `landing.jobs` (an off-host or non-https
|
||||
* URL drops the posting). It is the dedup key and is display-only.
|
||||
* - company: derived from the URL slug (see companyFromUrl), falling back to
|
||||
* the portal entry name, then "Landing.jobs".
|
||||
* - location: first `locations[]` entry as "city, country_code"; "Remote" is
|
||||
* appended when `remote` is true.
|
||||
* - postedAt: `published_at` (else `created_at`) ISO date → epoch ms.
|
||||
*
|
||||
* @param {any} j
|
||||
* @param {string} [fallbackCompany]
|
||||
* @returns {{ title: string, url: string, company: string, location: string, postedAt?: number } | null}
|
||||
*/
|
||||
export function normalizeLandingJob(j, fallbackCompany) {
|
||||
if (!j || typeof j !== 'object') return null;
|
||||
|
||||
const title = typeof j.title === 'string' ? j.title.trim() : '';
|
||||
if (!title) return null;
|
||||
|
||||
let url = '';
|
||||
const rawUrl = typeof j.url === 'string' ? j.url.trim() : '';
|
||||
if (rawUrl) {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol === 'https:' && parsed.hostname === TRUSTED_HOST) url = parsed.href;
|
||||
} catch {
|
||||
// malformed URL → leave url = '' → dropped below
|
||||
}
|
||||
}
|
||||
if (!url) return null;
|
||||
|
||||
const company =
|
||||
companyFromUrl(url) ||
|
||||
(typeof fallbackCompany === 'string' && fallbackCompany.trim() ? fallbackCompany.trim() : 'Landing.jobs');
|
||||
|
||||
const first = Array.isArray(j.locations) && j.locations[0] && typeof j.locations[0] === 'object' ? j.locations[0] : {};
|
||||
const city = typeof first.city === 'string' ? first.city.trim() : '';
|
||||
const country = typeof first.country_code === 'string' ? first.country_code.trim() : '';
|
||||
const base = [city, country].filter(Boolean).join(', ');
|
||||
const location = [base, j.remote === true ? 'Remote' : ''].filter(Boolean).join(', ');
|
||||
|
||||
/** @type {{ title: string, url: string, company: string, location: string, postedAt?: number }} */
|
||||
const job = { title, url, company, location };
|
||||
const postedAt = toEpochMs(j.published_at) ?? toEpochMs(j.created_at);
|
||||
if (postedAt !== undefined) job.postedAt = postedAt;
|
||||
return job;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'landingjobs',
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
assertLandingUrl(FEED_URL);
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(FEED_URL, { redirect: 'error' });
|
||||
if (!Array.isArray(json)) {
|
||||
throw new Error(
|
||||
`landingjobs: unexpected API response — expected a JSON array, got ${json === null ? 'null' : typeof json}`,
|
||||
);
|
||||
}
|
||||
const fallbackCompany = entry?.name;
|
||||
return json.map(j => normalizeLandingJob(j, fallbackCompany)).filter(Boolean);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Lever provider — hits the public postings endpoint.
|
||||
// Auto-detects from careers_url pattern `https://jobs.lever.co/<slug>`.
|
||||
|
||||
function resolveApiUrl(entry) {
|
||||
const url = entry.careers_url || '';
|
||||
const match = url.match(/jobs\.lever\.co\/([^/?#]+)/);
|
||||
if (!match) return null;
|
||||
return `https://api.lever.co/v0/postings/${match[1]}`;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'lever',
|
||||
|
||||
detect(entry) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
return apiUrl ? { url: apiUrl } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
if (!apiUrl) throw new Error(`lever: cannot derive API URL for ${entry.name}`);
|
||||
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
|
||||
if (!Array.isArray(json)) return [];
|
||||
return json.map(j => ({
|
||||
title: j.text || '',
|
||||
url: j.hostedUrl || '',
|
||||
company: entry.name,
|
||||
location: j.categories?.location || '',
|
||||
// Lever's v0 postings list ships the full description for free (same
|
||||
// payload, no per-job request) — enables scan.mjs content_filter.
|
||||
description: typeof j.descriptionPlain === 'string' ? j.descriptionPlain : '',
|
||||
postedAt: typeof j.createdAt === 'number' ? j.createdAt : undefined,
|
||||
}));
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,211 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
import { execFile } from 'child_process';
|
||||
import { realpathSync } from 'fs';
|
||||
import { resolve, sep } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { promisify } from 'util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const LOCAL_PARSER_TIMEOUT_MS = 20_000;
|
||||
const LOCAL_PARSER_MAX_BUFFER_BYTES = 2_000_000;
|
||||
|
||||
// `parser.command` / `parser.script` come from portals.yml, which on a shared or
|
||||
// template config is not fully trusted. The command must be a known interpreter
|
||||
// or a file inside this project — never an arbitrary binary like `rm` or `curl`.
|
||||
const PROJECT_ROOT = realpathSync(resolve(fileURLToPath(new URL('..', import.meta.url))));
|
||||
const ALLOWED_INTERPRETERS = new Set(['python3', 'python', 'node', 'deno', 'bun', 'sh', 'bash']);
|
||||
|
||||
// `{careers_url}` and `{company}` are interpolated into the parser's argv. Validate
|
||||
// them so an interpolated value can never be read as a CLI flag (argument injection).
|
||||
function safeCareersUrl(value) {
|
||||
if (!value) return '';
|
||||
let url;
|
||||
try {
|
||||
url = new URL(String(value));
|
||||
} catch {
|
||||
throw new Error(`local-parser: careers_url is not a valid URL: ${value}`);
|
||||
}
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
||||
throw new Error(`local-parser: careers_url must be http(s): ${value}`);
|
||||
}
|
||||
return url.href;
|
||||
}
|
||||
|
||||
function safeCompany(value) {
|
||||
if (!value) return '';
|
||||
const name = String(value).trim();
|
||||
// execFile passes args verbatim (no shell), so the only injection risk is a
|
||||
// value that begins like a CLI flag.
|
||||
if (name.startsWith('-')) {
|
||||
throw new Error(`local-parser: company name cannot start with '-': ${value}`);
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
// Only validate a placeholder's value when the arg actually uses it — a fixed
|
||||
// `parser.script` must not be rejected because some unrelated `{company}` value
|
||||
// has punctuation it never sees.
|
||||
function expandParserArg(value, entry) {
|
||||
let out = String(value);
|
||||
if (out.includes('{careers_url}')) out = out.replaceAll('{careers_url}', safeCareersUrl(entry.careers_url));
|
||||
if (out.includes('{company}')) out = out.replaceAll('{company}', safeCompany(entry.name));
|
||||
return out;
|
||||
}
|
||||
|
||||
function getParserScriptPath(entry) {
|
||||
const parser = entry.parser || {};
|
||||
if (parser.script) return expandParserArg(parser.script, entry);
|
||||
|
||||
const args = Array.isArray(parser.args) ? parser.args : [];
|
||||
const scriptArg = args.find(arg => {
|
||||
const value = String(arg);
|
||||
return !value.startsWith('-') && /\.(py|mjs|js|sh)$/.test(value);
|
||||
});
|
||||
|
||||
return scriptArg ? expandParserArg(scriptArg, entry) : null;
|
||||
}
|
||||
|
||||
function buildParserArgs(entry) {
|
||||
const parser = entry.parser || {};
|
||||
const args = [];
|
||||
|
||||
if (parser.script) args.push(parser.script);
|
||||
if (Array.isArray(parser.args)) args.push(...parser.args);
|
||||
|
||||
return args.map(arg => expandParserArg(arg, entry));
|
||||
}
|
||||
|
||||
// Resolve a configured path and confirm it stays inside the project tree.
|
||||
function resolveInsideRoot(rawPath) {
|
||||
const resolved = realpathSync(resolve(PROJECT_ROOT, String(rawPath)));
|
||||
if (resolved !== PROJECT_ROOT && !resolved.startsWith(PROJECT_ROOT + sep)) {
|
||||
throw new Error(`local-parser: path escapes the project root: ${rawPath}`);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
// The command is either a whitelisted interpreter (resolved via PATH) or a script
|
||||
// that lives inside the repo. Anything else is rejected.
|
||||
function resolveCommand(command) {
|
||||
const value = String(command || '');
|
||||
if (!value) throw new Error('local-parser: parser.command is required');
|
||||
if (!value.includes('/') && ALLOWED_INTERPRETERS.has(value)) return value;
|
||||
return resolveInsideRoot(value);
|
||||
}
|
||||
|
||||
// Validate the whole invocation and return what to spawn. Throws on anything unsafe.
|
||||
function resolveInvocation(entry) {
|
||||
const rawCommand = String(entry.parser?.command || '');
|
||||
const command = resolveCommand(rawCommand);
|
||||
const args = buildParserArgs(entry);
|
||||
const scriptPath = getParserScriptPath(entry);
|
||||
|
||||
const usesInterpreter = !rawCommand.includes('/') && ALLOWED_INTERPRETERS.has(rawCommand);
|
||||
if (usesInterpreter) {
|
||||
// A whitelisted interpreter must run an in-repo script as its FIRST argument.
|
||||
// Anything before the script is an interpreter option (node --eval / --require,
|
||||
// python -c, …) that could execute arbitrary code, so require the script to lead.
|
||||
if (!scriptPath) throw new Error('local-parser: interpreter command requires an in-repo parser script');
|
||||
resolveInsideRoot(scriptPath);
|
||||
if (args[0] !== scriptPath) {
|
||||
throw new Error('local-parser: the parser script must be the interpreter\'s first argument');
|
||||
}
|
||||
} else if (scriptPath) {
|
||||
// command is an in-repo file; keep any detected script path inside the repo too.
|
||||
resolveInsideRoot(scriptPath);
|
||||
}
|
||||
|
||||
return { command, args };
|
||||
}
|
||||
|
||||
function normalizeJobUrl(rawUrl, baseUrl) {
|
||||
if (!rawUrl) return '';
|
||||
try {
|
||||
return new URL(String(rawUrl).trim(), baseUrl || undefined).href;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeLocation(value) {
|
||||
if (!value) return '';
|
||||
if (Array.isArray(value)) return value.map(normalizeLocation).filter(Boolean).join(', ');
|
||||
if (typeof value === 'object') return value.name || value.text || '';
|
||||
return String(value).trim();
|
||||
}
|
||||
|
||||
function normalizeParserJob(job, entry) {
|
||||
if (!job || typeof job !== 'object') return null;
|
||||
|
||||
const title = String(job.title || job.name || '').trim();
|
||||
const url = normalizeJobUrl(
|
||||
job.url || job.jobUrl || job.job_url || job.applyUrl || job.apply_url,
|
||||
entry.careers_url,
|
||||
);
|
||||
if (!title || !url) return null;
|
||||
|
||||
return {
|
||||
title,
|
||||
url,
|
||||
company: String(job.company || entry.name || '').trim(),
|
||||
location: normalizeLocation(job.location || job.locations),
|
||||
};
|
||||
}
|
||||
|
||||
async function runLocalParser(entry) {
|
||||
const parser = entry.parser || {};
|
||||
const { command, args } = resolveInvocation(entry);
|
||||
const timeout = Number(parser.timeout_ms || LOCAL_PARSER_TIMEOUT_MS);
|
||||
const maxBuffer = Number(parser.max_buffer_bytes || LOCAL_PARSER_MAX_BUFFER_BYTES);
|
||||
|
||||
// cwd is pinned to the project root so a relative script arg resolves to the
|
||||
// same file resolveInvocation() validated, regardless of the caller's cwd.
|
||||
const { stdout } = await execFileAsync(command, args, {
|
||||
cwd: PROJECT_ROOT,
|
||||
timeout,
|
||||
maxBuffer,
|
||||
windowsHide: true,
|
||||
});
|
||||
|
||||
let payload;
|
||||
try {
|
||||
payload = JSON.parse(stdout);
|
||||
} catch {
|
||||
throw new Error('local parser returned invalid JSON');
|
||||
}
|
||||
|
||||
const rawJobs = Array.isArray(payload) ? payload : payload.jobs || payload.results;
|
||||
if (!Array.isArray(rawJobs)) {
|
||||
throw new Error('local parser JSON must be an array or contain jobs[]/results[]');
|
||||
}
|
||||
|
||||
return rawJobs
|
||||
.map(job => normalizeParserJob(job, entry))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'local-parser',
|
||||
|
||||
detect(entry) {
|
||||
if (!entry.parser?.command) return null;
|
||||
|
||||
// An invocation we can't safely resolve (unknown command, out-of-repo or
|
||||
// missing script, inline-code flags) is not runnable — skip it.
|
||||
try {
|
||||
resolveInvocation(entry);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { url: entry.careers_url || 'local-parser' };
|
||||
},
|
||||
|
||||
async fetch(entry) {
|
||||
return runLocalParser(entry);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,157 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// NoDesk provider - board-wide RSS feed
|
||||
// (https://nodesk.co/remote-jobs/index.xml). The feed is public, no-auth,
|
||||
// and XML, so it is parsed in-process with the same tiny tag extractor
|
||||
// approach as providers/personio.mjs rather than adding an XML dependency.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: nodesk`.
|
||||
|
||||
const FEED_URL = 'https://nodesk.co/remote-jobs/index.xml';
|
||||
const TRUSTED_HOST = 'nodesk.co';
|
||||
|
||||
/** @param {string} url */
|
||||
function assertNodeskUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`nodesk: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`nodesk: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST) {
|
||||
throw new Error(`nodesk: untrusted hostname "${parsed.hostname}" - must be ${TRUSTED_HOST}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
function fallbackCompany(entry) {
|
||||
return typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : 'NoDesk';
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'nodesk',
|
||||
|
||||
detect(entry) {
|
||||
return entry?.provider === 'nodesk' ? { url: FEED_URL } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const feedUrl = assertNodeskUrl(FEED_URL);
|
||||
// redirect:'error' prevents SSRF via server-side redirects; combined with
|
||||
// assertNodeskUrl above it keeps the request pinned to nodesk.co.
|
||||
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
|
||||
return parseNodeskFeed(text, fallbackCompany(entry));
|
||||
},
|
||||
};
|
||||
|
||||
function fromCodePoint(cp) {
|
||||
try {
|
||||
return String.fromCodePoint(cp);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// Decode the XML entities that appear in RSS text: numeric (& / ')
|
||||
// and the named five. Numeric forms are decoded first; & is decoded LAST
|
||||
// so a literal "&lt;" yields "<" rather than over-decoding to "<".
|
||||
function decodeXmlEntities(s) {
|
||||
return s
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
|
||||
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
|
||||
function extractText(inner) {
|
||||
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
|
||||
if (cdata) return cdata[1].trim();
|
||||
return decodeXmlEntities(inner).trim();
|
||||
}
|
||||
|
||||
// Extract the text of the first <tag>...</tag> in a block. Returns '' when absent.
|
||||
function tagText(block, tag) {
|
||||
const m = block.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, 'i'));
|
||||
return m ? extractText(m[1]) : '';
|
||||
}
|
||||
|
||||
// Keep only absolute HTTPS links hosted on the trusted board domain.
|
||||
function cleanUrl(value) {
|
||||
if (!value) return '';
|
||||
const trimmed = value.trim();
|
||||
try {
|
||||
const parsed = new URL(trimmed);
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
const trusted = host === TRUSTED_HOST || host.endsWith(`.${TRUSTED_HOST}`);
|
||||
return parsed.protocol === 'https:' && trusted ? parsed.href : '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// NoDesk encodes the company in the RSS title as "Role at Company".
|
||||
function splitTitle(rawTitle, defaultCompany) {
|
||||
const text = rawTitle.trim();
|
||||
const lower = text.toLowerCase();
|
||||
const idx = lower.lastIndexOf(' at ');
|
||||
if (idx > 0) {
|
||||
const title = text.slice(0, idx).trim();
|
||||
const company = text.slice(idx + 4).trim();
|
||||
if (title && company) return { title, company };
|
||||
}
|
||||
return { title: text, company: defaultCompany };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse NoDesk's public RSS jobs feed. Exported for unit tests.
|
||||
*
|
||||
* Shape: `<rss><channel><item>...</item>...</channel></rss>`. Each item
|
||||
* exposes `<title>`, `<link>`, and `<pubDate>`. NoDesk currently encodes the
|
||||
* company inside the title as `Role at Company`; there is no dedicated
|
||||
* location tag in the feed, so location stays empty unless the feed evolves.
|
||||
*
|
||||
* @param {string} xml - raw RSS feed body
|
||||
* @param {string} [defaultCompany] - fallback company for unsplittable titles
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parseNodeskFeed(xml, defaultCompany = 'NoDesk') {
|
||||
if (typeof xml !== 'string') return [];
|
||||
const fallback = typeof defaultCompany === 'string' && defaultCompany.trim() ? defaultCompany.trim() : 'NoDesk';
|
||||
const jobs = [];
|
||||
const blocks = xml.match(/<item\b[^>]*>[\s\S]*?<\/item>/gi) || [];
|
||||
|
||||
for (const item of blocks) {
|
||||
const url = cleanUrl(tagText(item, 'link'));
|
||||
if (!url) continue;
|
||||
|
||||
const rawTitle = tagText(item, 'title');
|
||||
if (!rawTitle) continue;
|
||||
|
||||
const { title, company } = splitTitle(rawTitle, fallback);
|
||||
const postedAt = toEpochMs(tagText(item, 'pubDate'));
|
||||
const job = {
|
||||
title,
|
||||
company,
|
||||
location: '',
|
||||
url,
|
||||
};
|
||||
if (postedAt !== undefined) job.postedAt = postedAt;
|
||||
jobs.push(job);
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// NoFluffJobs provider — hits the public search posting API.
|
||||
// It intentionally returns only the core scanner job fields; richer skill and
|
||||
// salary metadata can be added later if the provider contract is expanded.
|
||||
|
||||
const ALLOWED_HOSTS = new Set(['nofluffjobs.com']);
|
||||
const API_URL = 'https://nofluffjobs.com/api/search/posting';
|
||||
const JOB_BASE = 'https://nofluffjobs.com/pl/job/';
|
||||
const PAGE_SIZE = 20;
|
||||
const MAX_PAGES = 5;
|
||||
|
||||
function assertNoFluffUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`nofluffjobs: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`nofluffjobs: URL must use HTTPS: ${url}`);
|
||||
if (!ALLOWED_HOSTS.has(parsed.hostname)) {
|
||||
throw new Error(`nofluffjobs: untrusted hostname "${parsed.hostname}" — must be nofluffjobs.com`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function detectUrl(entry) {
|
||||
const url = entry.api || entry.careers_url || '';
|
||||
if (typeof url !== 'string' || !url.trim()) return null;
|
||||
try {
|
||||
return { url: assertNoFluffUrl(url).href };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeLocation(posting) {
|
||||
const parts = [];
|
||||
if (posting?.fullyRemote || posting?.location?.fullyRemote) parts.push('Remote');
|
||||
if (Array.isArray(posting?.location?.places)) {
|
||||
for (const place of posting.location.places) {
|
||||
const city = String(place?.city || '').trim();
|
||||
const province = String(place?.province || '').trim();
|
||||
const country = String(place?.country?.name || '').trim();
|
||||
if (city) parts.push(city);
|
||||
else if (province) parts.push(province);
|
||||
else if (country) parts.push(country);
|
||||
}
|
||||
}
|
||||
return [...new Set(parts.filter(Boolean))].join(', ');
|
||||
}
|
||||
|
||||
function postedAtMillis(value) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? number : undefined;
|
||||
}
|
||||
|
||||
function buildRequest(entry, pageTo) {
|
||||
const apiUrl = assertNoFluffUrl(entry.api || API_URL);
|
||||
apiUrl.pathname = '/api/search/posting';
|
||||
apiUrl.search = '';
|
||||
apiUrl.searchParams.set('sort', 'newest');
|
||||
apiUrl.searchParams.set('withSalaryMatch', 'true');
|
||||
apiUrl.searchParams.set('pageTo', String(pageTo));
|
||||
apiUrl.searchParams.set('pageSize', String(Number(entry.page_size || PAGE_SIZE)));
|
||||
apiUrl.searchParams.set('salaryCurrency', String(entry.currency || 'PLN').toUpperCase());
|
||||
apiUrl.searchParams.set('salaryPeriod', String(entry.salary_period || 'month'));
|
||||
apiUrl.searchParams.set('region', String(entry.region || 'pl'));
|
||||
apiUrl.searchParams.set('language', String(entry.language || 'pl-PL'));
|
||||
|
||||
const rawSearch = String(entry.search || entry.raw_search || '').trim();
|
||||
const body = rawSearch
|
||||
? {
|
||||
criteria: '',
|
||||
url: { searchParam: rawSearch },
|
||||
rawSearch,
|
||||
pageSize: Number(entry.page_size || PAGE_SIZE),
|
||||
withSalaryMatch: true,
|
||||
}
|
||||
: {
|
||||
criteriaSearch: {
|
||||
country: [],
|
||||
withSalaryMatch: [],
|
||||
city: [],
|
||||
more: [],
|
||||
employment: [],
|
||||
requirement: [],
|
||||
salary: [],
|
||||
jobPosition: [],
|
||||
applicationStatus: [],
|
||||
province: [],
|
||||
company: [],
|
||||
id: [],
|
||||
category: [],
|
||||
keyword: [],
|
||||
jobLanguage: [],
|
||||
seniority: [],
|
||||
},
|
||||
pageSize: Number(entry.page_size || PAGE_SIZE),
|
||||
withSalaryMatch: true,
|
||||
};
|
||||
|
||||
return { url: apiUrl.href, body };
|
||||
}
|
||||
|
||||
export function parseNoFluffJobsResponse(json) {
|
||||
if (!json || !Array.isArray(json.postings)) {
|
||||
throw new Error(`nofluffjobs: unexpected API response — expected { postings: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
|
||||
}
|
||||
return json.postings
|
||||
.filter(posting => posting && typeof posting === 'object')
|
||||
.map(posting => {
|
||||
const title = String(posting.title || '').trim();
|
||||
const company = String(posting.name || '').trim();
|
||||
const slug = String(posting.url || posting.id || '').trim();
|
||||
if (!title || !slug) return null;
|
||||
return {
|
||||
title,
|
||||
url: `${JOB_BASE}${slug}`,
|
||||
company,
|
||||
location: normalizeLocation(posting),
|
||||
postedAt: postedAtMillis(posting.posted),
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'nofluffjobs',
|
||||
|
||||
detect(entry) {
|
||||
return detectUrl(entry);
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
if (!detectUrl(entry)) throw new Error('nofluffjobs: careers_url or api must be a trusted nofluffjobs.com URL');
|
||||
|
||||
const maxPages = Number(entry.max_pages || MAX_PAGES);
|
||||
const jobs = [];
|
||||
const seen = new Set();
|
||||
|
||||
for (let pageTo = 1; pageTo <= maxPages; pageTo++) {
|
||||
const { url, body } = buildRequest(entry, pageTo);
|
||||
const json = await ctx.fetchJson(url, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(body),
|
||||
headers: {
|
||||
accept: 'application/json, text/plain, */*',
|
||||
'content-type': 'application/infiniteSearch+json',
|
||||
},
|
||||
redirect: 'error',
|
||||
});
|
||||
for (const job of parseNoFluffJobsResponse(json)) {
|
||||
if (seen.has(job.url)) continue;
|
||||
seen.add(job.url);
|
||||
jobs.push(job);
|
||||
}
|
||||
const totalPages = Number(json?.totalPages || 0);
|
||||
if (totalPages && pageTo >= totalPages) break;
|
||||
if (json.postings.length === 0) break;
|
||||
}
|
||||
|
||||
return jobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,169 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Personio provider — hits the public, no-auth XML jobs feed at
|
||||
// `https://<slug>.jobs.personio.de/xml` (common across DACH/EU companies).
|
||||
// Auto-detects from a `<slug>.jobs.personio.(de|com)` careers host like
|
||||
// workable/recruitee. Per-tenant subdomains are the variable part, so the
|
||||
// SSRF defence is an anchored host regex rather than a static allowlist.
|
||||
//
|
||||
// The feed is a flat, well-defined XML document, so it is parsed in-process
|
||||
// with a tiny tag extractor (no new dependency — the repo ships none for XML).
|
||||
|
||||
const PERSONIO_HOST_RE = /^[a-z0-9][a-z0-9-]*\.jobs\.personio\.(de|com)$/;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertPersonioUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`personio: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`personio: URL must use HTTPS: ${url}`);
|
||||
if (!PERSONIO_HOST_RE.test(parsed.hostname))
|
||||
throw new Error(`personio: untrusted hostname "${parsed.hostname}" — must match <slug>.jobs.personio.(de|com)`);
|
||||
return url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the tenant host (e.g. `acme.jobs.personio.de`) from a careers_url.
|
||||
* Returns null for non-Personio or malformed URLs.
|
||||
* @param {import('./_types.js').PortalEntry} entry
|
||||
*/
|
||||
function resolveHost(entry) {
|
||||
const raw = typeof entry.careers_url === 'string' ? entry.careers_url : '';
|
||||
if (!raw) return null;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (parsed.protocol !== 'https:') return null;
|
||||
if (!PERSONIO_HOST_RE.test(parsed.hostname)) return null;
|
||||
return parsed.hostname;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'personio',
|
||||
|
||||
detect(entry) {
|
||||
const host = resolveHost(entry);
|
||||
return host ? { url: `https://${host}/xml` } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const host = resolveHost(entry);
|
||||
if (!host) throw new Error(`personio: cannot derive feed URL for ${entry.name}`);
|
||||
const feedUrl = `https://${host}/xml`;
|
||||
assertPersonioUrl(feedUrl);
|
||||
// redirect:'error' prevents SSRF via server-side redirects; combined with
|
||||
// assertPersonioUrl above it guarantees the final hostname stays in-domain.
|
||||
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
|
||||
return parsePersonioXml(text, entry.name, host);
|
||||
},
|
||||
};
|
||||
|
||||
function fromCodePoint(cp) {
|
||||
try {
|
||||
return String.fromCodePoint(cp);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// Decode the XML entities that appear in Personio job text: numeric (& /
|
||||
// ') and the named five. Numeric forms are decoded first; & is decoded
|
||||
// LAST so a literal "&lt;" yields "<" rather than over-decoding to "<".
|
||||
function decodeXmlEntities(s) {
|
||||
return s
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
|
||||
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
|
||||
function extractText(inner) {
|
||||
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
|
||||
if (cdata) return cdata[1].trim();
|
||||
return decodeXmlEntities(inner).trim();
|
||||
}
|
||||
|
||||
// Extract the text of the first <tag>…</tag> in a block. Returns '' when absent.
|
||||
function tagText(block, tag) {
|
||||
const m = block.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`));
|
||||
return m ? extractText(m[1]) : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Personio's public XML jobs feed. Exported for unit tests.
|
||||
*
|
||||
* Shape: `<workzag-jobs><position>…</position>…</workzag-jobs>`, each position
|
||||
* carrying `<id>`, `<name>`, `<office>` (+ optional `<additionalOffices><office>`),
|
||||
* and `<createdAt>` (ISO 8601). The feed has NO per-job URL, so it is built from
|
||||
* the already-validated tenant host: `https://<host>/job/<id>`.
|
||||
*
|
||||
* - title: `<name>` (required — positions without one are dropped).
|
||||
* - url: `https://<host>/job/<id>` — only when `<id>` is a plain integer, so a
|
||||
* malformed id can never inject into the URL. url is the dedup key; a position
|
||||
* without a usable id is dropped.
|
||||
* - location: every `<office>` in the block (primary + additionalOffices),
|
||||
* de-duplicated, joined with ", ".
|
||||
* - postedAt: `<createdAt>` → epoch ms (omitted when unparseable/absent).
|
||||
*
|
||||
* @param {string} xml — raw XML feed body
|
||||
* @param {string} companyName — value written into job.company
|
||||
* @param {string} host — validated tenant host, e.g. `acme.jobs.personio.de`
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parsePersonioXml(xml, companyName, host) {
|
||||
if (typeof xml !== 'string') return [];
|
||||
const jobs = [];
|
||||
// Strip every <jobDescriptions> subtree from the WHOLE feed before splitting
|
||||
// into <position> blocks: descriptions are free-text HTML that can carry a
|
||||
// literal "</position>" which would otherwise truncate the non-greedy block
|
||||
// match. It also drops the per-section <name>/<value> pairs whose nested
|
||||
// <name> would race the position's own <name> (same for any other scalar tag).
|
||||
const stripped = xml.replace(/<jobDescriptions\b[^>]*>[\s\S]*?<\/jobDescriptions>/gi, '');
|
||||
const blocks = stripped.match(/<position\b[^>]*>[\s\S]*?<\/position>/g) || [];
|
||||
for (const scalar of blocks) {
|
||||
const title = tagText(scalar, 'name');
|
||||
if (!title) continue;
|
||||
|
||||
const id = tagText(scalar, 'id');
|
||||
if (!/^\d+$/.test(id)) continue; // need a clean numeric id to build the url
|
||||
|
||||
// Collect every <office> (primary + additionalOffices), de-dupe, join.
|
||||
const offices = [];
|
||||
const seen = new Set();
|
||||
for (const om of scalar.matchAll(/<office\b[^>]*>([\s\S]*?)<\/office>/g)) {
|
||||
const name = extractText(om[1]);
|
||||
if (name && !seen.has(name)) {
|
||||
seen.add(name);
|
||||
offices.push(name);
|
||||
}
|
||||
}
|
||||
|
||||
jobs.push({
|
||||
title,
|
||||
url: `https://${host}/job/${id}`,
|
||||
location: offices.join(', '),
|
||||
company: companyName,
|
||||
postedAt: toEpochMs(tagText(scalar, 'createdAt')),
|
||||
});
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Pinpoint provider — hits the public per-tenant postings.json feed.
|
||||
// Auto-detects from careers_url pattern `https://<slug>.pinpointhq.com`.
|
||||
//
|
||||
// Pinpoint's postings.json is public, zero-auth and returns every active
|
||||
// posting for the tenant in a single call:
|
||||
// https://<slug>.pinpointhq.com/postings.json
|
||||
// Response shape: { data: [ { title, url, path, location: { name, city,
|
||||
// province, ... }, job: { department, division, ... }, compensation, ... } ] }
|
||||
//
|
||||
// Per-tenant subdomains are the variable part — SSRF defence uses a regex
|
||||
// match on `<safe-slug>.pinpointhq.com` rather than a static allowlist, the
|
||||
// same approach as the recruitee provider.
|
||||
|
||||
// The tenant label must be a valid DNS label: it may contain hyphens but must
|
||||
// not start or end with one (so `acme-.pinpointhq.com` is rejected). The
|
||||
// optional trailing group keeps single-character labels (e.g. `a.pinpointhq.com`)
|
||||
// valid. detect() and fetch() both route through this constant via
|
||||
// resolveApiUrl()/assertPinpointUrl(), so the stricter check applies everywhere.
|
||||
const PINPOINT_HOST_RE = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.pinpointhq\.com$/;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertPinpointUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`pinpoint: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`pinpoint: URL must use HTTPS: ${url}`);
|
||||
if (!PINPOINT_HOST_RE.test(parsed.hostname)) {
|
||||
throw new Error(`pinpoint: untrusted hostname "${parsed.hostname}" — must match <slug>.pinpointhq.com`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
function resolveApiUrl(entry) {
|
||||
const raw = typeof entry.careers_url === 'string' ? entry.careers_url : '';
|
||||
if (!raw) return null;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (parsed.protocol !== 'https:') return null;
|
||||
if (!PINPOINT_HOST_RE.test(parsed.hostname)) return null;
|
||||
return `https://${parsed.hostname}/postings.json`;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'pinpoint',
|
||||
|
||||
detect(entry) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
return apiUrl ? { url: apiUrl } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
if (!apiUrl) throw new Error(`pinpoint: cannot derive API URL for ${entry.name}`);
|
||||
assertPinpointUrl(apiUrl);
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
|
||||
return parsePinpointResponse(json, entry.name);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a Pinpoint /postings.json response. Exported for unit tests.
|
||||
*
|
||||
* Pinpoint returns:
|
||||
* { data: [{ title, url, path, location: { name, city, province, ... }, ... }] }
|
||||
*
|
||||
* Field mapping → the normalized Job shape:
|
||||
* - title: `title`, trimmed.
|
||||
* - url: `url` — an absolute posting URL on the tenant's own
|
||||
* `<slug>.pinpointhq.com` host. It is display-only (written to the
|
||||
* pipeline and scan history, never server-fetched here), so it is
|
||||
* not host-locked; the requirement is a well-formed `https:` URL.
|
||||
* - location: prefer the display `location.name`; otherwise assemble from
|
||||
* `location.city` / `location.province`.
|
||||
* - company: the source carries no per-posting company name (each feed is a
|
||||
* single tenant), so it is sourced from the portal entry name,
|
||||
* mirroring the recruitee provider.
|
||||
*
|
||||
* Rows missing a usable title or a valid `https:` URL are dropped — an empty
|
||||
* URL would corrupt the scanner's URL-based dedup key.
|
||||
*
|
||||
* @param {any} json
|
||||
* @param {string} companyName
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string}>}
|
||||
*/
|
||||
export function parsePinpointResponse(json, companyName) {
|
||||
const postings = json?.data;
|
||||
if (!Array.isArray(postings)) return [];
|
||||
return postings
|
||||
.map(j => {
|
||||
const title = typeof j?.title === 'string' ? j.title.trim() : '';
|
||||
if (!title) return null;
|
||||
|
||||
// url: require a well-formed https: URL (display-only; see doc above).
|
||||
let url = '';
|
||||
const rawUrl = typeof j?.url === 'string' ? j.url.trim() : '';
|
||||
if (rawUrl) {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol === 'https:') url = parsed.href;
|
||||
} catch {
|
||||
// malformed URL → leave url = '' → dropped below
|
||||
}
|
||||
}
|
||||
if (!url) return null;
|
||||
|
||||
// location: prefer the display name, else assemble from city/province.
|
||||
const loc = j?.location && typeof j.location === 'object' ? j.location : {};
|
||||
const name = typeof loc.name === 'string' ? loc.name.trim() : '';
|
||||
const city = typeof loc.city === 'string' ? loc.city.trim() : '';
|
||||
const province = typeof loc.province === 'string' ? loc.province.trim() : '';
|
||||
const location = name || [city, province].filter(Boolean).join(', ');
|
||||
|
||||
return { title, url, location, company: companyName };
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Recruitee provider — hits the public per-tenant offers API.
|
||||
// Auto-detects from careers_url pattern `https://<slug>.recruitee.com`.
|
||||
// Per-tenant subdomains are the variable part — SSRF defence uses a
|
||||
// regex match on `<safe-slug>.recruitee.com` rather than a static
|
||||
// allowlist.
|
||||
|
||||
const RECRUITEE_HOST_RE = /^[a-z0-9][a-z0-9-]*\.recruitee\.com$/;
|
||||
|
||||
function assertRecruiteeUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`recruitee: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`recruitee: URL must use HTTPS: ${url}`);
|
||||
if (!RECRUITEE_HOST_RE.test(parsed.hostname)) {
|
||||
throw new Error(`recruitee: untrusted hostname "${parsed.hostname}" — must match <slug>.recruitee.com`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
function resolveApiUrl(entry) {
|
||||
const raw = typeof entry.careers_url === 'string' ? entry.careers_url : '';
|
||||
if (!raw) return null;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (parsed.protocol !== 'https:') return null;
|
||||
if (!RECRUITEE_HOST_RE.test(parsed.hostname)) return null;
|
||||
return `https://${parsed.hostname}/api/offers/`;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'recruitee',
|
||||
|
||||
detect(entry) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
return apiUrl ? { url: apiUrl } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
if (!apiUrl) throw new Error(`recruitee: cannot derive API URL for ${entry.name}`);
|
||||
assertRecruiteeUrl(apiUrl);
|
||||
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
|
||||
return parseRecruiteeResponse(json, entry.name);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a Recruitee /api/offers/ response. Exported for unit tests.
|
||||
*
|
||||
* Recruitee returns:
|
||||
* { offers: [{ title, careers_url?, url?, city?, country?, remote?, location? }] }
|
||||
*
|
||||
* - url: prefer `careers_url`, fall back to `url`. Recruitee tenants commonly
|
||||
* serve postings on their own custom domain (e.g. `careers.hostaway.com`),
|
||||
* so this URL is NOT host-locked to `*.recruitee.com`. Unlike the API
|
||||
* endpoint, the per-offer URL is display-only — it is written to the pipeline
|
||||
* and scan history but never server-fetched here, so the SSRF rationale does
|
||||
* not apply. It is sourced from the already-validated tenant API response.
|
||||
* Requirement: a well-formed `https:` URL; a non-HTTPS or malformed URL is
|
||||
* dropped (empty string returned per the Job contract).
|
||||
* - location: prefer the explicit `location` field; else assemble from
|
||||
* city/country, appending "Remote" when `remote` is true.
|
||||
*
|
||||
* @param {any} json
|
||||
* @param {string} companyName
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string}>}
|
||||
*/
|
||||
export function parseRecruiteeResponse(json, companyName) {
|
||||
const offers = json?.offers;
|
||||
if (!Array.isArray(offers)) return [];
|
||||
return offers.map(j => {
|
||||
const city = j.city || '';
|
||||
const country = j.country || '';
|
||||
const remote = j.remote ? 'Remote' : '';
|
||||
const location = j.location || [city, country, remote].filter(Boolean).join(', ');
|
||||
|
||||
// Resolve offer URL. Recruitee tenants commonly publish postings on their
|
||||
// own custom domain (e.g. careers.hostaway.com), so the per-offer URL is
|
||||
// NOT host-locked to *.recruitee.com — it is display-only (recorded in the
|
||||
// pipeline/history, never server-fetched here) and comes from the already-
|
||||
// validated tenant API response. Require only a well-formed https: URL;
|
||||
// a non-https or malformed URL is dropped (empty string per the Job contract).
|
||||
let url = '';
|
||||
const rawUrl = j.careers_url || j.url || '';
|
||||
if (typeof rawUrl === 'string' && rawUrl) {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol === 'https:') {
|
||||
url = parsed.href;
|
||||
}
|
||||
} catch {
|
||||
// malformed URL → leave url = ''
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title: j.title || '',
|
||||
url,
|
||||
location,
|
||||
company: companyName,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// RemoteOK provider — board-wide aggregator feed (https://remoteok.com/api).
|
||||
// Returns the latest ~100 remote postings as a JSON array; index 0 is a
|
||||
// {last_updated, legal} metadata object and is skipped. scan.mjs applies the
|
||||
// configured title_filter / location_filter to the returned rows.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: remoteok`.
|
||||
// RemoteOK API ToS asks for a follow link-back when republishing — N/A for
|
||||
// private scanning, but don't redistribute this feed publicly without it.
|
||||
|
||||
const FEED_URL = 'https://remoteok.com/api';
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'remoteok',
|
||||
|
||||
/**
|
||||
* Fetches and normalizes postings from the RemoteOK public feed.
|
||||
* @param {{ name?: string }} entry - The job_boards entry being processed.
|
||||
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
|
||||
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>}
|
||||
*/
|
||||
async fetch(entry, ctx) {
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const data = await ctx.fetchJson(FEED_URL, { redirect: 'error' });
|
||||
if (!Array.isArray(data)) {
|
||||
throw new Error(`remoteok: unexpected API response — expected a JSON array, got ${data === null ? 'null' : typeof data}`);
|
||||
}
|
||||
|
||||
return data
|
||||
.filter(j => j && typeof j === 'object'
|
||||
&& typeof j.position === 'string' && j.position.trim() !== ''
|
||||
&& typeof j.url === 'string' && /^https?:\/\//i.test(j.url.trim()))
|
||||
.map(j => ({
|
||||
title: j.position.trim(),
|
||||
url: j.url.trim(),
|
||||
company: typeof j.company === 'string' && j.company.trim() ? j.company.trim() : (entry.name || 'RemoteOK'),
|
||||
location: typeof j.location === 'string' ? j.location.trim() : '',
|
||||
}));
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,42 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Remotive provider — board-wide aggregator feed
|
||||
// (https://remotive.com/api/remote-jobs). Returns { jobs: [...] }. The full
|
||||
// feed (no ?search=) is fetched so scan.mjs's title_filter can gate on the
|
||||
// configured AI/ML titles; the feed's own ?search= is too narrow (a substring
|
||||
// match that misses e.g. "ML Engineer").
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: remotive`.
|
||||
|
||||
const FEED_URL = 'https://remotive.com/api/remote-jobs';
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'remotive',
|
||||
|
||||
/**
|
||||
* Fetches and normalizes postings from the Remotive public feed.
|
||||
* @param {{ name?: string }} entry - The job_boards entry being processed.
|
||||
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
|
||||
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>}
|
||||
*/
|
||||
async fetch(entry, ctx) {
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(FEED_URL, { redirect: 'error' });
|
||||
if (!json || !Array.isArray(json.jobs)) {
|
||||
throw new Error(`remotive: unexpected API response — expected { jobs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
|
||||
}
|
||||
|
||||
return json.jobs
|
||||
.filter(j => j && typeof j === 'object'
|
||||
&& typeof j.title === 'string' && j.title.trim() !== ''
|
||||
&& typeof j.url === 'string' && /^https?:\/\//i.test(j.url.trim()))
|
||||
.map(j => ({
|
||||
title: j.title.trim(),
|
||||
url: j.url.trim(),
|
||||
company: typeof j.company_name === 'string' && j.company_name.trim() ? j.company_name.trim() : (entry.name || 'Remotive'),
|
||||
location: typeof j.candidate_required_location === 'string' ? j.candidate_required_location.trim() : '',
|
||||
}));
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,136 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Rippling provider — hits the public per-tenant ATS board API.
|
||||
// Auto-detects from a careers_url like `https://ats.rippling.com/<slug>/jobs`
|
||||
// (the `<slug>` is the first path segment). Rippling's board API is public and
|
||||
// zero-auth:
|
||||
// https://api.rippling.com/platform/api/ats/v1/board/<slug>/jobs
|
||||
// Response shape: a JSON ARRAY of
|
||||
// { uuid, name, department: { id, label }, url, workLocation: { id, label } }
|
||||
//
|
||||
// The careers host (`ats.rippling.com`) and the API host (`api.rippling.com`)
|
||||
// are both fixed; the per-tenant slug is the only variable part. It is extracted
|
||||
// from the careers URL path and constrained to a safe token so it cannot inject
|
||||
// extra path segments or traversal into the API URL.
|
||||
|
||||
const CAREERS_HOST = 'ats.rippling.com';
|
||||
const API_HOST = 'api.rippling.com';
|
||||
const API_BASE = `https://${API_HOST}/platform/api/ats/v1/board`;
|
||||
const SLUG_RE = /^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$/;
|
||||
|
||||
/**
|
||||
* Resolve the tenant slug (e.g. `just-appraised-jobs`) from a careers_url.
|
||||
* Returns null for non-Rippling, malformed, or unsafe-slug URLs.
|
||||
* @param {import('./_types.js').PortalEntry} entry
|
||||
*/
|
||||
function resolveSlug(entry) {
|
||||
const raw = typeof entry.careers_url === 'string' ? entry.careers_url : '';
|
||||
if (!raw) return null;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (parsed.protocol !== 'https:') return null;
|
||||
if (parsed.hostname !== CAREERS_HOST) return null;
|
||||
const segment = parsed.pathname.split('/').filter(Boolean)[0] || '';
|
||||
if (!SLUG_RE.test(segment)) return null;
|
||||
return segment;
|
||||
}
|
||||
|
||||
/** Build the board API URL for a validated slug. */
|
||||
function apiUrlForSlug(slug) {
|
||||
return `${API_BASE}/${encodeURIComponent(slug)}/jobs`;
|
||||
}
|
||||
|
||||
/** @param {string} url */
|
||||
function assertRipplingApiUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`rippling: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`rippling: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== API_HOST) {
|
||||
throw new Error(`rippling: untrusted hostname "${parsed.hostname}" — must be ${API_HOST}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'rippling',
|
||||
|
||||
detect(entry) {
|
||||
const slug = resolveSlug(entry);
|
||||
return slug ? { url: apiUrlForSlug(slug) } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const slug = resolveSlug(entry);
|
||||
if (!slug) throw new Error(`rippling: cannot derive API URL for ${entry.name}`);
|
||||
const apiUrl = apiUrlForSlug(slug);
|
||||
assertRipplingApiUrl(apiUrl);
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
|
||||
return parseRipplingResponse(json, entry.name);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a Rippling board API response. Exported for unit tests.
|
||||
*
|
||||
* The response is a top-level JSON ARRAY of postings. Field mapping → the
|
||||
* normalized Job shape:
|
||||
* - title: `name`, trimmed (postings without one are dropped).
|
||||
* - url: `url` — an absolute `https:` posting URL host-locked to
|
||||
* `ats.rippling.com` (Rippling always serves postings there, so an
|
||||
* off-host or non-https URL is untrusted and the posting is dropped).
|
||||
* It is the dedup key and is display-only (written to the
|
||||
* pipeline/history, never server-fetched here).
|
||||
* - company: the portal entry name (the feed is per-tenant and carries no
|
||||
* company field, same as recruitee).
|
||||
* - location: `workLocation.label` (e.g. "Remote (United States)"); falls back
|
||||
* to a bare string `workLocation`, else "".
|
||||
*
|
||||
* @param {any} json
|
||||
* @param {string} companyName
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string}>}
|
||||
*/
|
||||
export function parseRipplingResponse(json, companyName) {
|
||||
if (!Array.isArray(json)) return [];
|
||||
return json
|
||||
.map(j => {
|
||||
const title = typeof j?.name === 'string' ? j.name.trim() : '';
|
||||
if (!title) return null;
|
||||
|
||||
// url must be an absolute https posting link on ats.rippling.com — Rippling
|
||||
// always serves postings there (no custom-domain case), so an off-host URL
|
||||
// is untrusted and dropped. url is the dedup key.
|
||||
let url = '';
|
||||
const rawUrl = typeof j?.url === 'string' ? j.url.trim() : '';
|
||||
if (rawUrl) {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol === 'https:' && parsed.hostname === CAREERS_HOST) url = parsed.href;
|
||||
} catch {
|
||||
// malformed URL → leave url = '' → dropped below
|
||||
}
|
||||
}
|
||||
if (!url) return null;
|
||||
|
||||
const wl = j?.workLocation;
|
||||
const location =
|
||||
wl && typeof wl === 'object' && typeof wl.label === 'string'
|
||||
? wl.label.trim()
|
||||
: typeof wl === 'string'
|
||||
? wl.trim()
|
||||
: '';
|
||||
|
||||
return { title, url, location, company: companyName };
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// SmartRecruiters provider — hits the public postings API.
|
||||
// Auto-detects from careers_url pattern
|
||||
// `https://(careers|jobs).smartrecruiters.com/<slug>`. A tracked_companies
|
||||
// entry can also set `provider: smartrecruiters` explicitly to bypass
|
||||
// detection (useful when the public careers URL is a branded custom domain).
|
||||
|
||||
const ALLOWED_SMARTRECRUITERS_HOSTS = new Set(['api.smartrecruiters.com']);
|
||||
const SR_CAREERS_HOSTS = new Set(['careers.smartrecruiters.com', 'jobs.smartrecruiters.com']);
|
||||
const SR_PAGE_SIZE = 100;
|
||||
const SR_MAX_PAGES = 50; // safety cap (5000 postings @ 100/page)
|
||||
|
||||
function assertSmartRecruitersUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`smartrecruiters: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`smartrecruiters: URL must use HTTPS: ${url}`);
|
||||
if (!ALLOWED_SMARTRECRUITERS_HOSTS.has(parsed.hostname)) {
|
||||
throw new Error(`smartrecruiters: untrusted hostname "${parsed.hostname}" — must be one of: ${[...ALLOWED_SMARTRECRUITERS_HOSTS].join(', ')}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
function resolveSlug(entry) {
|
||||
// entry.api takes precedence over careers_url (mirrors greenhouse/ashby) so a
|
||||
// branded page (e.g. https://jobs.continental.com) can stay as careers_url
|
||||
// while the SmartRecruiters slug is pinned via
|
||||
// api: https://careers.smartrecruiters.com/<slug> in portals.yml.
|
||||
for (const raw of [entry.api, entry.careers_url]) {
|
||||
if (typeof raw !== 'string' || !raw) continue;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (parsed.protocol !== 'https:') continue;
|
||||
if (!SR_CAREERS_HOSTS.has(parsed.hostname)) continue;
|
||||
const slug = parsed.pathname.split('/').filter(Boolean)[0];
|
||||
if (slug) return slug;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildPostingsUrl(slug, offset = 0) {
|
||||
return `https://api.smartrecruiters.com/v1/companies/${slug}/postings?limit=${SR_PAGE_SIZE}&offset=${offset}&status=PUBLIC`;
|
||||
}
|
||||
|
||||
function resolveApiUrl(entry) {
|
||||
const slug = resolveSlug(entry);
|
||||
return slug ? buildPostingsUrl(slug, 0) : null;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'smartrecruiters',
|
||||
|
||||
detect(entry) {
|
||||
const apiUrl = resolveApiUrl(entry);
|
||||
return apiUrl ? { url: apiUrl } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const slug = resolveSlug(entry);
|
||||
if (!slug) throw new Error(`smartrecruiters: cannot derive API URL for ${entry.name}`);
|
||||
|
||||
const all = [];
|
||||
for (let page = 0; page < SR_MAX_PAGES; page++) {
|
||||
const apiUrl = buildPostingsUrl(slug, page * SR_PAGE_SIZE);
|
||||
assertSmartRecruitersUrl(apiUrl);
|
||||
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
|
||||
const parsed = parseSmartRecruitersResponse(json, entry.name);
|
||||
if (parsed.length === 0) break;
|
||||
all.push(...parsed);
|
||||
if (parsed.length < SR_PAGE_SIZE) break; // last page (short)
|
||||
}
|
||||
return all;
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a SmartRecruiters /postings response. Exported for unit tests.
|
||||
*
|
||||
* SmartRecruiters returns:
|
||||
* { content: [{ id, name, ref, location: { fullLocation?, city?, region?, country?, remote? } }] }
|
||||
*
|
||||
* - location: prefer `fullLocation`; else assemble from city/region/country
|
||||
* parts (skipping empties); append "Remote" when `location.remote` is true.
|
||||
* - url: `j.ref` is an `api.smartrecruiters.com/v1/companies/<slug>/postings/<id>`
|
||||
* URL — rewrite to the public `jobs.smartrecruiters.com/<slug>/postings/<id>`.
|
||||
* If `ref` is missing, synthesise a URL from the company slug + posting id.
|
||||
*
|
||||
* @param {any} json
|
||||
* @param {string} companyName
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string}>}
|
||||
*/
|
||||
export function parseSmartRecruitersResponse(json, companyName) {
|
||||
const items = json?.content;
|
||||
if (!Array.isArray(items)) return [];
|
||||
return items.map(j => {
|
||||
const loc = j.location || {};
|
||||
const fullLocation = loc.fullLocation || [loc.city, loc.region, loc.country].filter(Boolean).join(', ');
|
||||
const remote = loc.remote ? 'Remote' : '';
|
||||
const location = [fullLocation, remote].filter(Boolean).join(', ');
|
||||
const slugified = (j.name || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
||||
let url = '';
|
||||
if (typeof j.ref === 'string') {
|
||||
let parsedRef;
|
||||
try { parsedRef = new URL(j.ref); } catch { parsedRef = null; }
|
||||
if (parsedRef
|
||||
&& parsedRef.protocol === 'https:'
|
||||
&& parsedRef.hostname === 'api.smartrecruiters.com'
|
||||
&& parsedRef.pathname.startsWith('/v1/companies/')) {
|
||||
const restOfPath = parsedRef.pathname.slice('/v1/companies/'.length);
|
||||
url = `https://jobs.smartrecruiters.com/${restOfPath}`;
|
||||
}
|
||||
}
|
||||
if (!url && j.id) {
|
||||
const companySlug = (companyName || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
||||
if (companySlug) {
|
||||
url = `https://jobs.smartrecruiters.com/${companySlug}/${j.id}-${slugified}`;
|
||||
}
|
||||
}
|
||||
return { title: j.name || '', url, location, company: companyName };
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// softgarden provider — the hosted job widgets at
|
||||
// https://{tenant}.softgarden.io/{lang}/widgets/jobs (e.g. RENK:
|
||||
// renk-group.softgarden.io/de/widgets/jobs). Companies embed this Wicket page
|
||||
// as an iframe on their branded career sites; fetched directly it is a plain
|
||||
// server-rendered document that lists EVERY posting — no auth, no JS, no
|
||||
// pagination observed (filtering happens client-side via AJAX we don't need).
|
||||
//
|
||||
// One posting renders as:
|
||||
// <div class="matchElement" id="job_id_{id}">
|
||||
// <div class="matchValue date">04.07.26</div>
|
||||
// <div class="matchValue title">
|
||||
// <a href="../../job/{id}/{slug}?jobDbPVId=…&l=de">{Title}</a></div>
|
||||
// <div class="matchValue audience">…</div>
|
||||
// <div class="matchValue ProjectGeoLocationCity">…
|
||||
// <span class="location-view-item">{City}</span>…</div>
|
||||
// </div>
|
||||
//
|
||||
// The href is relative to the /{lang}/widgets/ path — we resolve it against
|
||||
// the widget URL so it lands on {origin}/job/{id}/{slug}?… (verified live).
|
||||
// The date is the locale's short form (de: D.M.YY with dots; en variants use
|
||||
// slashes M/D/YY) — same heuristic as the successfactors CSB parser.
|
||||
|
||||
const MAX_JOBS = 1000; // cap postings taken from one widget page
|
||||
|
||||
// Minimal HTML entity decoder — titles carry named (&) and numeric
|
||||
// (ü / ü) entities. Mirrors successfactors.mjs / dassault.mjs.
|
||||
const NAMED_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
|
||||
/** @param {string} s */
|
||||
function decodeEntities(s) {
|
||||
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (m, body) => {
|
||||
if (body[0] === '#') {
|
||||
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
||||
return Number.isInteger(code) && code >= 0 && code <= 0x10ffff ? String.fromCodePoint(code) : m;
|
||||
}
|
||||
return NAMED_ENTITIES[body.toLowerCase()] ?? m;
|
||||
});
|
||||
}
|
||||
|
||||
/** @param {string} s */
|
||||
function clean(s) {
|
||||
return decodeEntities(s.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
/** Resolve the widget URL: explicit /widgets/jobs URLs pass through, any other
|
||||
* *.softgarden.io URL defaults to the German jobs widget. */
|
||||
export function resolveWidgetUrl(entry) {
|
||||
const raw = entry.api || entry.careers_url || '';
|
||||
let u;
|
||||
try {
|
||||
u = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
|
||||
const host = u.host.toLowerCase();
|
||||
if (host !== 'softgarden.io' && !host.endsWith('.softgarden.io')) return null;
|
||||
if (/\/widgets\/jobs\/?$/.test(u.pathname)) return `${u.origin}${u.pathname.replace(/\/$/, '')}`;
|
||||
const lang = (u.pathname.match(/^\/([a-z]{2})(\/|$)/) || [])[1] || 'de';
|
||||
return `${u.origin}/${lang}/widgets/jobs`;
|
||||
}
|
||||
|
||||
// Widget dates are the locale's SHORT form: de widgets emit D.M.YY with dots,
|
||||
// en variants M/D/YY with slashes. Separator infers the field order; 2-digit
|
||||
// years are 20YY. Junk → undefined ("unknown date", never a bogus timestamp).
|
||||
/** @param {unknown} raw @returns {number | undefined} */
|
||||
export function parseSoftgardenDate(raw) {
|
||||
if (typeof raw !== 'string') return undefined;
|
||||
const s = raw.trim();
|
||||
let a, b, year;
|
||||
const slash = s.match(/^(\d{1,2})\/(\d{1,2})\/(\d{2,4})$/);
|
||||
const dot = s.match(/^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$/);
|
||||
if (slash) {
|
||||
[, a, b, year] = slash; // M/D/Y
|
||||
} else if (dot) {
|
||||
[, b, a, year] = dot; // D.M.Y → swap to month/day
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
const month = Number(a);
|
||||
const day = Number(b);
|
||||
let y = Number(year);
|
||||
if (y < 100) y += 2000;
|
||||
if (month < 1 || month > 12 || day < 1 || day > 31) return undefined;
|
||||
const ms = Date.UTC(y, month - 1, day);
|
||||
return Number.isFinite(ms) ? ms : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the widget page into raw {id, title, url, location, postedAt} records.
|
||||
* @param {string} html @param {string} widgetUrl
|
||||
*/
|
||||
export function parseWidget(html, widgetUrl) {
|
||||
if (typeof html !== 'string') return [];
|
||||
const out = [];
|
||||
const seen = new Set();
|
||||
const blocks = html.split(/<div class="matchElement" id="job_id_/).slice(1);
|
||||
for (const block of blocks) {
|
||||
const id = (block.match(/^(\d+)"/) || [])[1];
|
||||
if (!id || seen.has(id)) continue;
|
||||
const linkM = block.match(/<a href="([^"]*\/job\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/);
|
||||
if (!linkM) continue;
|
||||
const title = clean(linkM[2]);
|
||||
if (!title) continue;
|
||||
let url;
|
||||
try {
|
||||
// hrefs are relative to the /{lang}/widgets/ path ("../../job/…").
|
||||
url = new URL(decodeEntities(linkM[1]), widgetUrl).href;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const cities = [];
|
||||
const cityRe = /class="location-view-item"[^>]*>([\s\S]*?)<\/span>/g;
|
||||
let cm;
|
||||
while ((cm = cityRe.exec(block)) !== null) {
|
||||
const c = clean(cm[1]);
|
||||
if (c && !cities.includes(c)) cities.push(c);
|
||||
}
|
||||
const dateM = block.match(/class="matchValue date"[^>]*>([\s\S]*?)<\/div>/);
|
||||
seen.add(id);
|
||||
out.push({
|
||||
id,
|
||||
title,
|
||||
url,
|
||||
location: cities.join(' / '),
|
||||
postedAt: parseSoftgardenDate(dateM ? clean(dateM[1]) : undefined),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'softgarden',
|
||||
|
||||
detect(entry) {
|
||||
const url = entry.api || entry.careers_url || '';
|
||||
if (typeof url !== 'string') return null;
|
||||
return resolveWidgetUrl({ api: url }) ? { url } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const widgetUrl = resolveWidgetUrl(entry);
|
||||
if (!widgetUrl) throw new Error(`softgarden: cannot resolve widget URL for ${entry.name}`);
|
||||
|
||||
const html = await ctx.fetchText(widgetUrl, { headers: { accept: 'text/html' } });
|
||||
const rows = parseWidget(html, widgetUrl);
|
||||
const jobs = [];
|
||||
for (const row of rows) {
|
||||
const job = { title: row.title, url: row.url, company: entry.name, location: row.location };
|
||||
if (typeof row.postedAt === 'number') job.postedAt = row.postedAt;
|
||||
jobs.push(job);
|
||||
if (jobs.length >= MAX_JOBS) break;
|
||||
}
|
||||
return jobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,82 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// SolidJobs provider — hits the public offers API.
|
||||
// Auto-detects from careers_url pattern `https://solid.jobs/public-api/offers/<division>`.
|
||||
//
|
||||
// Available divisions: it, engineering, marketing, sales, hr, logistics, finances, other
|
||||
// API docs: https://solid.jobs/public-api/offers/{division}?campaign={campaign}
|
||||
|
||||
const ALLOWED_HOSTS = new Set(['solid.jobs']);
|
||||
|
||||
/**
|
||||
* Validates that the provided URL is a trusted SolidJobs API endpoint.
|
||||
* Enforces HTTPS protocol, strict hostname matching, and required path prefix.
|
||||
*
|
||||
* @param {string} url - The URL string to validate.
|
||||
* @returns {string} The validated URL string.
|
||||
* @throws {Error} If the URL is malformed, uses non-HTTPS, has an untrusted host, or wrong path.
|
||||
*/
|
||||
function assertUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`solidjobs: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`solidjobs: URL must use HTTPS: ${url}`);
|
||||
if (!ALLOWED_HOSTS.has(parsed.hostname))
|
||||
throw new Error(`solidjobs: untrusted hostname "${parsed.hostname}" — must be solid.jobs`);
|
||||
if (!parsed.pathname.startsWith('/public-api/offers/'))
|
||||
throw new Error(`solidjobs: URL path must start with /public-api/offers/: ${url}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'solidjobs',
|
||||
|
||||
/**
|
||||
* Attempts to detect if the provider can handle the given entry by checking the careers_url.
|
||||
* * @param {{ careers_url?: string, name?: string }} entry - The configuration entry.
|
||||
* @returns {{url: string} | null} An object with the matched URL, or null if not matched.
|
||||
*/
|
||||
detect(entry) {
|
||||
const url = entry.careers_url || '';
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (parsed.hostname === 'solid.jobs' && parsed.pathname.startsWith('/public-api/offers/'))
|
||||
return { url };
|
||||
} catch {}
|
||||
return null;
|
||||
},
|
||||
|
||||
/**
|
||||
* Fetches and normalizes job offers from the SolidJobs public API.
|
||||
* * @param {{ careers_url?: string, name: string }} entry - The configuration entry being processed.
|
||||
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
|
||||
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>} Array of parsed job offers.
|
||||
*/
|
||||
async fetch(entry, ctx) {
|
||||
const url = entry.careers_url;
|
||||
if (!url) throw new Error('solidjobs: careers_url required');
|
||||
assertUrl(url);
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(url, { redirect: 'error' });
|
||||
if (!json || !Array.isArray(json.jobs)) {
|
||||
throw new Error(`solidjobs: unexpected API response — expected { jobs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
|
||||
}
|
||||
|
||||
/** @type {Array<{ title?: string, url?: string, company?: string, locations?: string | string[] }>} */
|
||||
const jobs = json.jobs;
|
||||
|
||||
return jobs
|
||||
.filter(j => j && typeof j === 'object' && typeof j.url === 'string' && j.url.trim() !== '')
|
||||
.map(j => ({
|
||||
title: j.title || '',
|
||||
url: /** @type {string} */ (j.url || '').trim(),
|
||||
company: j.company || entry.name,
|
||||
location: Array.isArray(j.locations) ? j.locations.join(', ') : (typeof j.locations === 'string' ? j.locations : ''),
|
||||
}));
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,410 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// SAP SuccessFactors provider — Recruiting Marketing (RMK, ex-jobs2web) career
|
||||
// sites (Career Site Builder's branded job boards). These are the portals big
|
||||
// German industrials run at their own domains (jobs.zf.com, jobs.schaeffler.com,
|
||||
// jobs.hensoldt.net, jobs.sap.com, …) — all served by the same SF RMK backend.
|
||||
//
|
||||
// The RMK backend exposes a public, no-auth job-list fragment endpoint:
|
||||
// GET {origin}/tile-search-results/?startrow={N}
|
||||
// It returns an HTML fragment (not JSON) of <li class="job-tile job-id-{id}">
|
||||
// blocks, each carrying data-url (job path), the title link, and — when the
|
||||
// tenant configures it — a rendered city field. We parse those out; the scanner
|
||||
// contract only needs {title, url, company, location, postedAt}.
|
||||
//
|
||||
// Pagination: startrow is a plain row offset, and page size is tenant-specific
|
||||
// (Schaeffler serves 100/page, most others 25). We never assume a page size —
|
||||
// each loop advances startrow by the number of tiles actually returned, and a
|
||||
// per-id dedup set guards against overlap or a server that ignores the offset.
|
||||
//
|
||||
// Detection: branded RMK hosts (jobs.zf.com) contain no "successfactors" string,
|
||||
// so detect() only auto-claims literal *.successfactors.eu/.com and jobs2web
|
||||
// URLs. The branded portals are wired up with an explicit `provider:
|
||||
// successfactors` in portals.yml (which bypasses detect()); `api:` may override
|
||||
// careers_url when the public careers_url isn't the RMK origin.
|
||||
//
|
||||
// RMK carries no posting date in the list fragment, so postedAt is always
|
||||
// omitted (consumers treat an absent date as "unknown", never as stale).
|
||||
//
|
||||
// ── CSB (Career Site Builder) variant ────────────────────────────────────────
|
||||
// Newer SF tenants run the Career Site Builder "unified" search (jobs2web's
|
||||
// searchResultsUnify.js). Their /tile-search-results/ endpoint returns only a
|
||||
// 16-byte empty <!DOCTYPE html> shell — the jobs are loaded client-side from a
|
||||
// JSON API instead:
|
||||
// POST {origin}/services/recruiting/v1/jobs
|
||||
// Content-Type: application/json
|
||||
// {"keywords":"","locale":"de_DE","location":"","pageNumber":0,"sortBy":"recent"}
|
||||
// → {"jobSearchResult":[{"response":{id,unifiedStandardTitle,unifiedUrlTitle,
|
||||
// jobLocationShort:[…],unifiedStandardStart:"6/18/26",…}}], "totalJobs":N}
|
||||
// 10 results/page; pageNumber is 0-based. The detail URL the site itself builds
|
||||
// (from searchResultsUnify.js) is {origin}/job/{urlTitle}/{id}-{locale}.
|
||||
//
|
||||
// CRITICAL — locale gating: the API only returns a posting under a locale it was
|
||||
// published in (each record carries `supportedLocales`). The same tenant reports
|
||||
// wildly different totals per locale — MAN: de_DE=601 but en_US=8; TRATON: en_US
|
||||
// only; Hexagon: en_GB only (de_DE/en_US both 0). A hardcoded locale would miss
|
||||
// entire tenants, so we DISCOVER each tenant's advertised locales from the
|
||||
// language switcher on {origin}/search/ (the `locale=xx_XX` links), query every
|
||||
// one, and dedup by job id. CSB carries a real posting date, so postedAt is set.
|
||||
//
|
||||
// Strategy selection: `sfVariant: csb` on the portal entry forces CSB directly;
|
||||
// otherwise the RMK tile path runs first and, only if it yields zero postings
|
||||
// (the empty-shell signature), we fall back to CSB automatically.
|
||||
|
||||
const MAX_PAGES = 40; // safety cap on request count (RMK tile pagination)
|
||||
const MAX_JOBS = 1000; // cap total postings pulled per site (both strategies)
|
||||
|
||||
// CSB knobs.
|
||||
const CSB_PAGE_SIZE = 10; // fixed by the /services/recruiting/v1/jobs API
|
||||
const CSB_MAX_PAGES_PER_LOCALE = 100; // safety cap per locale (100*10 = 1000 postings)
|
||||
const CSB_MAX_LOCALES = 16; // guard against a tenant advertising an absurd locale list
|
||||
// Locales tried when /search/ discovery yields nothing (fetch failed / markup
|
||||
// changed). de_DE + en_US cover the DACH targets this provider is aimed at.
|
||||
const CSB_DEFAULT_LOCALES = ['de_DE', 'en_US'];
|
||||
// Query order: the DACH-relevant, usually job-rich locales first, so a bounded
|
||||
// probe (max_pages:1) hits real postings early and dedup keeps a sensible URL
|
||||
// locale for jobs published in several languages.
|
||||
const CSB_LOCALE_PRIORITY = ['de_DE', 'en_US', 'en_GB', 'en_EN'];
|
||||
|
||||
/** @param {import('./_types.js').PortalEntry} entry */
|
||||
function resolveConfig(entry) {
|
||||
const raw = entry.api || entry.careers_url || '';
|
||||
let u;
|
||||
try {
|
||||
u = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
|
||||
return {
|
||||
origin: u.origin,
|
||||
tileApi: `${u.origin}/tile-search-results/`,
|
||||
jobBase: u.origin,
|
||||
jobsApi: `${u.origin}/services/recruiting/v1/jobs`,
|
||||
searchPage: `${u.origin}/search/`,
|
||||
};
|
||||
}
|
||||
|
||||
// Minimal HTML entity decoder — titles carry named (&) and numeric
|
||||
// (ü / ü) entities. We only need the handful that show up in job
|
||||
// titles; anything else is left as-is.
|
||||
const NAMED_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
|
||||
/** @param {string} s */
|
||||
function decodeEntities(s) {
|
||||
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (m, body) => {
|
||||
if (body[0] === '#') {
|
||||
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
||||
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
|
||||
}
|
||||
return NAMED_ENTITIES[body.toLowerCase()] ?? m;
|
||||
});
|
||||
}
|
||||
|
||||
/** @param {string} s */
|
||||
function clean(s) {
|
||||
return decodeEntities(s.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
// RMK job paths encode the city as the leading slug segment:
|
||||
// /job/{City-Words}-{Title-Words}-{reqCode}/{id}/
|
||||
// When the tenant doesn't render a dedicated city field we recover the city by
|
||||
// stripping the title (and trailing code) off the front slug. We anchor on the
|
||||
// first two title words — a prefix the slug always reproduces verbatim, unlike
|
||||
// the title's tail where punctuation like "(m/w/d)" gets mangled.
|
||||
/** @param {string} dataUrl @param {string} title */
|
||||
export function cityFromSlug(dataUrl, title) {
|
||||
let path;
|
||||
try {
|
||||
path = decodeURIComponent(dataUrl);
|
||||
} catch {
|
||||
path = dataUrl;
|
||||
}
|
||||
const m = path.match(/\/job\/([^/]+)\//);
|
||||
if (!m) return '';
|
||||
const slug = m[1].toLowerCase();
|
||||
const words = title.toLowerCase().match(/[\p{L}\p{N}]+/gu);
|
||||
if (!words || !words.length) return '';
|
||||
// Anchor on the first two title words, but allow ANY run of non-alphanumerics
|
||||
// between them — in the slug those words may be joined by "-", "-amp-" (a
|
||||
// decoded "&"), or several hyphens, none of which survive the title's word
|
||||
// split. A rigid "word-word" anchor would miss "Program & Release" → "program-&-release".
|
||||
const esc = (w) => w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
let anchorRe;
|
||||
try {
|
||||
anchorRe = new RegExp(words.slice(0, 2).map(esc).join('[^\\p{L}\\p{N}]+'), 'u');
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
const hit = slug.match(anchorRe);
|
||||
if (!hit || hit.index === undefined || hit.index <= 0) return '';
|
||||
return slug
|
||||
.slice(0, hit.index)
|
||||
.split(/[^\p{L}\p{N}]+/u)
|
||||
.filter(Boolean)
|
||||
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
// Parse one page fragment into raw {id, title, url, city} records.
|
||||
// Each posting renders three responsive copies (desktop/tablet/mobile) inside
|
||||
// one <li>; a single <li> per id means the top-level dedup already collapses
|
||||
// them, and within a block the first match of each field wins.
|
||||
/** @param {string} htmlText @param {string} jobBase */
|
||||
export function parseTiles(htmlText, jobBase) {
|
||||
const out = [];
|
||||
const tileRe = /<li class="job-tile job-id-(\d+)\b[\s\S]*?<\/li>/g;
|
||||
let t;
|
||||
while ((t = tileRe.exec(htmlText)) !== null) {
|
||||
const id = t[1];
|
||||
const block = t[0];
|
||||
const urlM = block.match(/data-url="([^"]+)"/);
|
||||
if (!urlM) continue;
|
||||
const titleM = block.match(/class="jobTitle-link[^"]*"[^>]*>([\s\S]*?)<\/a>/);
|
||||
const title = titleM ? clean(titleM[1]) : '';
|
||||
if (!title) continue;
|
||||
// data-url is an HTML attribute, so a literal "&" in the path arrives as
|
||||
// &. Decode entities (but not percent-encoding — %28 etc. stays) before
|
||||
// both URL building and slug parsing, so the "amp" of & can't leak into
|
||||
// the recovered city.
|
||||
const path = decodeEntities(urlM[1]);
|
||||
// Anchor on id="…-section-city-value"> — the value div. The sibling label
|
||||
// span references the same id via aria-describedby, so a looser match would
|
||||
// swallow the "City" label text too.
|
||||
const cityM = block.match(/id="[^"]*-section-city-value">([\s\S]*?)<\/div>/);
|
||||
const city = cityM ? clean(cityM[1]) : cityFromSlug(path, title);
|
||||
const url = /^https?:\/\//i.test(path) ? path : jobBase + (path.startsWith('/') ? path : '/' + path);
|
||||
out.push({ id, title, url, location: city });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── CSB helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
// Pull the tenant's advertised locales from the {origin}/search/ page. The CSB
|
||||
// language switcher renders them as `/search/?q=&startrow=0&locale=xx_XX` links
|
||||
// (HTML-escaped, so `&` — the regex just anchors on `locale=`). Returns a
|
||||
// de-duped, priority-ordered list; empty when the markup carries none.
|
||||
/** @param {string} html */
|
||||
export function extractLocales(html) {
|
||||
const found = new Set();
|
||||
const re = /locale=([a-z]{2}_[A-Z]{2})\b/g;
|
||||
let m;
|
||||
while ((m = re.exec(html)) !== null) found.add(m[1]);
|
||||
const all = [...found];
|
||||
const prio = (loc) => {
|
||||
const i = CSB_LOCALE_PRIORITY.indexOf(loc);
|
||||
return i === -1 ? CSB_LOCALE_PRIORITY.length : i;
|
||||
};
|
||||
all.sort((a, b) => prio(a) - prio(b) || a.localeCompare(b));
|
||||
return all.slice(0, CSB_MAX_LOCALES);
|
||||
}
|
||||
|
||||
// CSB posting dates are rendered in the query locale's short format: US tenants
|
||||
// send M/D/YY with slashes ("6/18/26"), European (de_DE) tenants send D.M.YY
|
||||
// with dots ("20.11.23"). Infer the field order from the separator, treat a
|
||||
// 2-digit year as 20YY, and return epoch ms. Anything unexpected yields
|
||||
// undefined so a consumer sees "unknown date" rather than a bogus timestamp.
|
||||
/** @param {unknown} raw */
|
||||
export function parseCsbDate(raw) {
|
||||
if (typeof raw !== 'string') return undefined;
|
||||
const s = raw.trim();
|
||||
let a, b, year;
|
||||
let slash = s.match(/^(\d{1,2})\/(\d{1,2})\/(\d{2,4})$/);
|
||||
let dot = s.match(/^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$/);
|
||||
if (slash) {
|
||||
// M/D/Y
|
||||
[, a, b, year] = slash;
|
||||
} else if (dot) {
|
||||
// D.M.Y — swap to month/day
|
||||
[, b, a, year] = dot;
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
const month = Number(a);
|
||||
const day = Number(b);
|
||||
let y = Number(year);
|
||||
if (y < 100) y += 2000;
|
||||
if (month < 1 || month > 12 || day < 1 || day > 31) return undefined;
|
||||
const ms = Date.UTC(y, month - 1, day);
|
||||
return Number.isFinite(ms) ? ms : undefined;
|
||||
}
|
||||
|
||||
// jobLocationShort is an array of strings like "Karlovy Vary, CZE, 36004<br/>"
|
||||
// (one per work location). Strip the trailing <br/> and any other markup, drop
|
||||
// empties, and join multiple locations with " / ".
|
||||
/** @param {unknown} raw */
|
||||
export function cleanCsbLocation(raw) {
|
||||
const parts = Array.isArray(raw) ? raw : raw == null ? [] : [raw];
|
||||
const cleaned = [];
|
||||
for (const p of parts) {
|
||||
const s = clean(String(p));
|
||||
if (s && !cleaned.includes(s)) cleaned.push(s);
|
||||
}
|
||||
return cleaned.join(' / ');
|
||||
}
|
||||
|
||||
// Map one CSB jobs-API response to raw {id, title, url, location, postedAt}.
|
||||
// Records nest the useful fields under `.response`; a record without an id or a
|
||||
// title is skipped (can't build a stable dedup key or a meaningful listing).
|
||||
/** @param {any} json @param {{origin:string}} cfg @param {string} locale */
|
||||
export function parseCsbJobs(json, cfg, locale) {
|
||||
const list = Array.isArray(json?.jobSearchResult) ? json.jobSearchResult : [];
|
||||
const out = [];
|
||||
for (const item of list) {
|
||||
const r = item?.response;
|
||||
if (!r) continue;
|
||||
const id = r.id != null ? String(r.id) : '';
|
||||
const title = clean(String(r.unifiedStandardTitle || r.jobTitle || ''));
|
||||
if (!id || !title) continue;
|
||||
// The site builds the slug from urlTitle/unifiedUrlTitle. The slug is purely
|
||||
// cosmetic — the server keys only on the {id}-{locale} tail (a stub slug
|
||||
// resolves the same posting) — but tenants render it inconsistently: some
|
||||
// percent-encode it ("Oper%C3%A1tor-…"), others leave HTML entities in
|
||||
// ("Mergers-&-Acquisitions"). Decode entities, then drop the URL-
|
||||
// structural chars (& ? #) a raw entity would otherwise inject into the
|
||||
// path, leaving any existing %XX escapes untouched.
|
||||
const slug = decodeEntities(String(r.unifiedUrlTitle || r.urlTitle || 'job'))
|
||||
.replace(/[?#&]+/g, '-')
|
||||
.replace(/-{2,}/g, '-');
|
||||
const url = `${cfg.origin}/job/${slug}/${id}-${locale}`;
|
||||
out.push({
|
||||
id,
|
||||
title,
|
||||
url,
|
||||
location: cleanCsbLocation(r.jobLocationShort),
|
||||
postedAt: parseCsbDate(r.unifiedStandardStart),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Resolve the per-locale page cap: positive integer `max_pages`, else default. */
|
||||
function resolveCsbMaxPages(entry) {
|
||||
const v = entry?.max_pages;
|
||||
if (Number.isInteger(v) && v > 0) return Math.min(v, CSB_MAX_PAGES_PER_LOCALE);
|
||||
return CSB_MAX_PAGES_PER_LOCALE;
|
||||
}
|
||||
|
||||
// CSB strategy: discover locales, paginate the JSON jobs API per locale, dedup
|
||||
// by id across locales+pages. `total` from the first page bounds pagination;
|
||||
// an empty/short page also stops the loop.
|
||||
/** @param {import('./_types.js').PortalEntry} entry @param {any} cfg @param {import('./_types.js').Context} ctx */
|
||||
async function fetchCsb(entry, cfg, ctx) {
|
||||
let locales = CSB_DEFAULT_LOCALES;
|
||||
try {
|
||||
const html = await ctx.fetchText(cfg.searchPage, { redirect: 'error', headers: { accept: 'text/html' } });
|
||||
const discovered = extractLocales(html);
|
||||
if (discovered.length) locales = discovered;
|
||||
} catch {
|
||||
// Discovery is best-effort; fall back to the default locale set.
|
||||
}
|
||||
|
||||
const maxPages = resolveCsbMaxPages(entry);
|
||||
const jobs = [];
|
||||
const seen = new Set();
|
||||
|
||||
for (const locale of locales) {
|
||||
if (jobs.length >= MAX_JOBS) break;
|
||||
let total = null;
|
||||
for (let page = 0; page < maxPages; page++) {
|
||||
let json;
|
||||
try {
|
||||
json = await ctx.fetchJson(cfg.jobsApi, {
|
||||
method: 'POST',
|
||||
redirect: 'error',
|
||||
headers: { 'content-type': 'application/json', accept: 'application/json' },
|
||||
body: JSON.stringify({ keywords: '', locale, location: '', pageNumber: page, sortBy: 'recent' }),
|
||||
});
|
||||
} catch {
|
||||
break; // this locale failed (e.g. 307 legacy redirect) — try the next
|
||||
}
|
||||
if (total === null) {
|
||||
total = typeof json?.totalJobs === 'number' ? json.totalJobs : null;
|
||||
}
|
||||
// Page-fullness checks below use the raw response count, not rows.length:
|
||||
// parseCsbJobs drops id/title-less records, and a full API page with a
|
||||
// few dropped records would otherwise look "short" and stop pagination
|
||||
// before reaching later pages that still hold valid jobs.
|
||||
const rawCount = Array.isArray(json?.jobSearchResult) ? json.jobSearchResult.length : 0;
|
||||
if (rawCount === 0) break;
|
||||
const rows = parseCsbJobs(json, cfg, locale);
|
||||
|
||||
for (const row of rows) {
|
||||
if (seen.has(row.id)) continue;
|
||||
seen.add(row.id);
|
||||
const job = { title: row.title, url: row.url, company: entry.name, location: row.location };
|
||||
if (typeof row.postedAt === 'number') job.postedAt = row.postedAt;
|
||||
jobs.push(job);
|
||||
if (jobs.length >= MAX_JOBS) break;
|
||||
}
|
||||
if (jobs.length >= MAX_JOBS) break;
|
||||
// Stop once we've covered the reported total, or on a short page.
|
||||
if (total !== null && (page + 1) * CSB_PAGE_SIZE >= total) break;
|
||||
if (rawCount < CSB_PAGE_SIZE) break;
|
||||
}
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
|
||||
// RMK strategy: the original /tile-search-results/ HTML-fragment scraper.
|
||||
/** @param {import('./_types.js').PortalEntry} entry @param {any} cfg @param {import('./_types.js').Context} ctx */
|
||||
async function fetchRmk(entry, cfg, ctx) {
|
||||
const jobs = [];
|
||||
const seen = new Set();
|
||||
let startrow = 0;
|
||||
for (let page = 0; page < MAX_PAGES; page++) {
|
||||
const htmlText = await ctx.fetchText(`${cfg.tileApi}?startrow=${startrow}`, {
|
||||
redirect: 'error',
|
||||
headers: { accept: 'text/html' },
|
||||
});
|
||||
const tiles = parseTiles(htmlText, cfg.jobBase);
|
||||
if (tiles.length === 0) break;
|
||||
|
||||
let fresh = 0;
|
||||
for (const tile of tiles) {
|
||||
if (seen.has(tile.id)) continue;
|
||||
seen.add(tile.id);
|
||||
fresh++;
|
||||
jobs.push({
|
||||
title: tile.title,
|
||||
url: tile.url,
|
||||
company: entry.name,
|
||||
location: tile.location,
|
||||
});
|
||||
}
|
||||
// No new ids this page → server ignored the offset (or we've looped). Stop.
|
||||
if (fresh === 0) break;
|
||||
if (jobs.length >= MAX_JOBS) break;
|
||||
startrow += tiles.length;
|
||||
}
|
||||
// The cap is checked between pages, so the last page can overshoot it.
|
||||
return jobs.slice(0, MAX_JOBS);
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'successfactors',
|
||||
|
||||
detect(entry) {
|
||||
const url = entry.api || entry.careers_url || '';
|
||||
if (/successfactors\.(eu|com)|jobs2web\.com/i.test(url)) return { url };
|
||||
return null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const cfg = resolveConfig(entry);
|
||||
if (!cfg) throw new Error(`successfactors: cannot resolve origin for ${entry.name}`);
|
||||
|
||||
// Explicit opt-in goes straight to CSB. RMK tenants (the majority) run the
|
||||
// tile scraper; only when it comes back empty — the CSB empty-shell
|
||||
// signature — do we auto-fall back to the JSON API, so an unflagged CSB
|
||||
// tenant still works and a genuinely-empty RMK board costs one extra probe.
|
||||
if (String(entry.sfVariant || '').toLowerCase() === 'csb') {
|
||||
return fetchCsb(entry, cfg, ctx);
|
||||
}
|
||||
const rmkJobs = await fetchRmk(entry, cfg, ctx);
|
||||
if (rmkJobs.length > 0) return rmkJobs;
|
||||
return fetchCsb(entry, cfg, ctx);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,200 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Teamtailor provider — per-tenant public RSS feed.
|
||||
//
|
||||
// Every Teamtailor career site exposes a zero-auth XML jobs feed at
|
||||
// `https://<slug>.teamtailor.com/jobs.rss`. The feed is public and no-auth, so
|
||||
// it is parsed in-process with the same tiny tag extractor as
|
||||
// providers/nodesk.mjs / providers/personio.mjs rather than adding an XML
|
||||
// dependency.
|
||||
//
|
||||
// Auto-detects `https://<slug>.teamtailor.com/...` careers URLs. Many tenants
|
||||
// front the same feed on a branded domain (e.g. careers.acme.com also serves
|
||||
// /jobs.rss), so an entry with an explicit `provider: teamtailor` may point its
|
||||
// `careers_url` (or `api:`) at that branded host and it will be used directly.
|
||||
//
|
||||
// SSRF stance mirrors the other providers: auto-detection stays pinned to
|
||||
// `*.teamtailor.com` so untrusted careers URLs can never steer the fetch at an
|
||||
// arbitrary host; a branded host is honored only when the user opted in with an
|
||||
// explicit `provider: teamtailor`. Either way the fetch is HTTPS-only with
|
||||
// `redirect: 'error'`. Job `<link>`s (branded domains) are emitted as-is and
|
||||
// never fetched.
|
||||
|
||||
const TEAMTAILOR_HOST_RE = /^([a-z0-9](?:[a-z0-9-]*[a-z0-9])?)\.teamtailor\.com$/i;
|
||||
|
||||
/**
|
||||
* Validate a feed URL before fetching. Always HTTPS-only. The hostname is
|
||||
* pinned to `*.teamtailor.com` for auto-detected entries; an explicit
|
||||
* `provider: teamtailor` entry may use its configured branded host.
|
||||
* @param {string} url
|
||||
* @param {{ explicit?: boolean }} [opts]
|
||||
*/
|
||||
function assertFeedUrl(url, { explicit = false } = {}) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`teamtailor: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`teamtailor: URL must use HTTPS: ${url}`);
|
||||
if (!explicit && !TEAMTAILOR_HOST_RE.test(parsed.hostname)) {
|
||||
throw new Error(`teamtailor: untrusted hostname "${parsed.hostname}" — must be <slug>.teamtailor.com (or set "provider: teamtailor" to use a branded careers domain)`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
// Derive the RSS feed URL from a tracked_companies entry by normalizing any
|
||||
// path on the configured host to /jobs.rss. Auto-detection (explicit=false)
|
||||
// only claims *.teamtailor.com hosts; an explicit `provider: teamtailor` entry
|
||||
// (explicit=true) may use a branded careers host. Returns null otherwise.
|
||||
/**
|
||||
* @param {import('./_types.js').PortalEntry} entry
|
||||
* @param {{ explicit?: boolean }} [opts]
|
||||
*/
|
||||
function resolveFeedUrl(entry, { explicit = false } = {}) {
|
||||
const raw = entry?.api || entry?.careers_url || '';
|
||||
if (typeof raw !== 'string' || !raw) return null;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (parsed.protocol !== 'https:') return null;
|
||||
if (!explicit && !TEAMTAILOR_HOST_RE.test(parsed.hostname)) return null;
|
||||
return `https://${parsed.hostname}/jobs.rss`;
|
||||
}
|
||||
|
||||
function fallbackCompany(entry) {
|
||||
return typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : 'Teamtailor';
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'teamtailor',
|
||||
|
||||
detect(entry) {
|
||||
// Auto-detection never claims a branded host — only *.teamtailor.com.
|
||||
const url = resolveFeedUrl(entry, { explicit: false });
|
||||
return url ? { url } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const explicit = entry?.provider === 'teamtailor';
|
||||
const feedUrl = resolveFeedUrl(entry, { explicit });
|
||||
if (!feedUrl) throw new Error(`teamtailor: cannot derive jobs.rss URL for ${fallbackCompany(entry)}`);
|
||||
assertFeedUrl(feedUrl, { explicit });
|
||||
// redirect:'error' prevents SSRF via server-side redirects; combined with
|
||||
// assertFeedUrl above it keeps the request pinned to the trusted host
|
||||
// (*.teamtailor.com, or the branded host the user explicitly configured).
|
||||
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
|
||||
return parseTeamtailorFeed(text, fallbackCompany(entry));
|
||||
},
|
||||
};
|
||||
|
||||
function fromCodePoint(cp) {
|
||||
try {
|
||||
return String.fromCodePoint(cp);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// Decode the XML entities that appear in RSS text: numeric (& / ')
|
||||
// and the named five. Numeric forms are decoded first; & is decoded LAST
|
||||
// so a literal "&lt;" yields "<" rather than over-decoding to "<".
|
||||
function decodeXmlEntities(s) {
|
||||
return s
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
|
||||
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
|
||||
function extractText(inner) {
|
||||
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
|
||||
if (cdata) return cdata[1].trim();
|
||||
return decodeXmlEntities(inner).trim();
|
||||
}
|
||||
|
||||
// Extract the text of the first <tag>...</tag> in a block. Returns '' when
|
||||
// absent. Tag names may contain a namespace colon (e.g. tt:city).
|
||||
function tagText(block, tag) {
|
||||
const m = block.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, 'i'));
|
||||
return m ? extractText(m[1]) : '';
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
// Keep only absolute HTTPS links. Teamtailor job URLs frequently live on a
|
||||
// branded custom domain, so — unlike the feed host — the link host is not
|
||||
// pinned; we only require a well-formed https URL.
|
||||
function cleanUrl(value) {
|
||||
if (!value) return '';
|
||||
try {
|
||||
const parsed = new URL(value.trim());
|
||||
return parsed.protocol === 'https:' ? parsed.href : '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// Build a human location from the first <tt:location> (city, country). Falls
|
||||
// back to "Remote" when the posting carries no place but is flagged remote.
|
||||
function resolveLocation(item) {
|
||||
const city = tagText(item, 'tt:city');
|
||||
const country = tagText(item, 'tt:country');
|
||||
const place = [city, country].filter(Boolean).join(', ');
|
||||
if (place) return place;
|
||||
const remote = tagText(item, 'remoteStatus').toLowerCase();
|
||||
return remote === 'fully' || remote === 'temporary' ? 'Remote' : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a Teamtailor public RSS jobs feed. Exported for unit tests.
|
||||
*
|
||||
* Shape: `<rss><channel><item>...</item>...</channel></rss>`. Each item
|
||||
* exposes `<title>`, `<link>`, `<pubDate>`, an optional `<remoteStatus>`, and
|
||||
* a `tt:` locations block (`<tt:city>`, `<tt:country>`). The company is not in
|
||||
* the item, so the tracked_companies `name` is used.
|
||||
*
|
||||
* @param {string} xml - raw RSS feed body
|
||||
* @param {string} [defaultCompany] - company label for every posting
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parseTeamtailorFeed(xml, defaultCompany = 'Teamtailor') {
|
||||
if (typeof xml !== 'string') return [];
|
||||
const company = typeof defaultCompany === 'string' && defaultCompany.trim() ? defaultCompany.trim() : 'Teamtailor';
|
||||
const jobs = [];
|
||||
const blocks = xml.match(/<item\b[^>]*>[\s\S]*?<\/item>/gi) || [];
|
||||
|
||||
for (const item of blocks) {
|
||||
const url = cleanUrl(tagText(item, 'link'));
|
||||
if (!url) continue;
|
||||
|
||||
const title = tagText(item, 'title');
|
||||
if (!title) continue;
|
||||
|
||||
const postedAt = toEpochMs(tagText(item, 'pubDate'));
|
||||
const job = {
|
||||
title,
|
||||
company,
|
||||
location: resolveLocation(item),
|
||||
url,
|
||||
};
|
||||
if (postedAt !== undefined) job.postedAt = postedAt;
|
||||
jobs.push(job);
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Telegram public-channel provider — reads the public web preview at
|
||||
// https://t.me/s/{channel}, the same lightweight HTML page Telegram serves
|
||||
// for link-preview embeds. No Bot API, no MTProto login, no credentials:
|
||||
// pure HTTP + regex parsing, same zero-token shape as every other provider.
|
||||
//
|
||||
// Limitations (accepted trade-off for zero-setup):
|
||||
// - Public channels only (no invite-only groups — those need a logged-in
|
||||
// MTProto client, a much heavier and more sensitive integration).
|
||||
// - Returns roughly the ~20 most recent posts per scan (no pagination).
|
||||
// Fine for periodic scanning as long as the channel doesn't outpace the
|
||||
// scan interval.
|
||||
// - Job fields (title/company/location) are heuristically extracted from
|
||||
// free-form message text — channels have no fixed schema. Expect noise;
|
||||
// tune title_filter/content_filter per channel same as any other source.
|
||||
// - When a post links out to the real source (a career page, hh.ru/geekjob.ru
|
||||
// posting, an inline "Подробнее"/"Откликнуться" button, etc.), that link
|
||||
// becomes the job's `url` instead of the Telegram post itself — it's
|
||||
// usually the fuller, more authoritative JD, and it lets the normal
|
||||
// liveness checker (Apply-control detection) work as intended. A link
|
||||
// that only opens inside the Telegram client (a t.me URL, including a bot
|
||||
// deep link like Runello's) is never usable this way and is skipped; the
|
||||
// post's own /s/ permalink is the fallback when no real link is found.
|
||||
//
|
||||
// Portal entry fields:
|
||||
// channel — Telegram username without '@', e.g. "jsjobru" (required
|
||||
// unless careers_url is a bare https://t.me/<channel> link,
|
||||
// which detect() also accepts)
|
||||
|
||||
const TRUSTED_HOST = 't.me';
|
||||
const CHANNEL_RE = /^[A-Za-z][A-Za-z0-9_]{4,31}$/;
|
||||
const POST_RE = /data-post="([a-zA-Z0-9_]+)\/(\d+)"/g;
|
||||
|
||||
// Some channels post digest/roundup messages ("Подборка вакансий недели")
|
||||
// that link to many jobs rather than describing one — not a postable offer.
|
||||
// \b doesn't work with Cyrillic (JS regex \b is defined over ASCII \w only),
|
||||
// so the boundary is a negative lookahead for another letter/digit instead.
|
||||
const NON_JOB_TITLE_RE = /^(подборка|дайджест|топ[\s-]|итоги|обзор)(?![\p{L}\p{N}])/iu;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertTelegramUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`telegram: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`telegram: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST)
|
||||
throw new Error(`telegram: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
function channelFromCareersUrl(careersUrl) {
|
||||
if (typeof careersUrl !== 'string') return '';
|
||||
try {
|
||||
const parsed = new URL(careersUrl);
|
||||
if (parsed.hostname !== 't.me' && parsed.hostname !== 'telegram.me') return '';
|
||||
const seg = parsed.pathname.replace(/^\//, '').split('/')[0] || '';
|
||||
return seg;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function resolveChannel(entry) {
|
||||
const raw = typeof entry?.channel === 'string' && entry.channel.trim()
|
||||
? entry.channel.trim().replace(/^@/, '')
|
||||
: channelFromCareersUrl(entry?.careers_url);
|
||||
return raw;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
function fromCodePoint(cp) {
|
||||
try {
|
||||
return String.fromCodePoint(cp);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(s) {
|
||||
return s
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
|
||||
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
// Convert a message-text block's inner HTML (line breaks + inline tags like
|
||||
// <a>/<b>/<i>/<span> for links/emphasis/hashtags) into plain text.
|
||||
function htmlBlockToText(html) {
|
||||
const withBreaks = html
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<\/(p|div)>/gi, '\n');
|
||||
const stripped = withBreaks.replace(/<[^>]+>/g, '');
|
||||
return decodeHtmlEntities(stripped).replace(/\n{3,}/g, '\n\n').trim();
|
||||
}
|
||||
|
||||
// First non-empty line, with leading emoji/bullet clutter stripped, as the
|
||||
// job title heuristic. Channels commonly lead posts with the role name.
|
||||
// Some channels pack "Title | Location | Company #hashtags url" onto that
|
||||
// single line instead of using separate lines — a real job title essentially
|
||||
// never contains a literal " | ", so cutting there discards the rest as noise
|
||||
// rather than dragging hashtags/links into the title.
|
||||
function extractTitle(text) {
|
||||
const firstLine = text.split('\n').map(l => l.trim()).find(l => l.length > 0) || '';
|
||||
const firstSegment = firstLine.split(/\s\|\s/)[0].trim();
|
||||
const cleaned = firstSegment.replace(/^[^\p{L}\p{N}]+/u, '').trim();
|
||||
return (cleaned || firstSegment).slice(0, 200);
|
||||
}
|
||||
|
||||
// Looks for a line that STARTS with "Label:" (RU/EN variants), anchored to
|
||||
// line-start so a section header like "О компании:" ("About the company:")
|
||||
// never matches the "компания" label — it starts with "О", not the label
|
||||
// itself. Returns '' if no such line exists.
|
||||
function extractField(text, labelAlternatives) {
|
||||
const re = new RegExp(`^(?:${labelAlternatives.join('|')})\\s*[:\\-–]\\s*(.+)$`, 'im');
|
||||
const m = text.match(re);
|
||||
return m ? m[1].trim() : '';
|
||||
}
|
||||
|
||||
// Second-line markers that mean "this is NOT a company name" — location pins,
|
||||
// salary lines, work-format tags, or a bullet from a multi-job digest post.
|
||||
// Different channels put wildly different things on line 2; without this
|
||||
// guard those get misread as a "company".
|
||||
const NON_COMPANY_LINE2_RE = /^(?:от|до|from|гибрид|удал[её]нн|офис|remote|hybrid|on-?site|🗺|📍|🔹|💵|💰)/i;
|
||||
|
||||
// Fallback when there's no explicit "Company:"/"Location:" line: many
|
||||
// channels follow "{Title}\n{Company}, {City}" — try the second non-empty
|
||||
// line, split on the first comma. Conservative on purpose: skipped if that
|
||||
// line looks like another labeled field, a hashtag/mention, a known
|
||||
// non-company marker (above), too long to be a short "Name, City" pair, or
|
||||
// truncated mid-sentence by Telegram's own preview ellipsis — all signs it's
|
||||
// prose (an "About us" blurb), not a clean company/city pair.
|
||||
function extractCompanyLocationFromLine2(text) {
|
||||
const lines = text.split('\n').map(l => l.trim()).filter(l => l.length > 0);
|
||||
const line2 = lines[1];
|
||||
if (!line2 || line2.includes(':') || /^[#@]/.test(line2)) return { company: '', location: '' };
|
||||
if (NON_COMPANY_LINE2_RE.test(line2)) return { company: '', location: '' };
|
||||
if (line2.length > 60 || /[…]\s*$/.test(line2)) return { company: '', location: '' };
|
||||
const parts = line2.split(',').map(p => p.trim()).filter(Boolean);
|
||||
if (parts.length === 1) return { company: line2, location: '' };
|
||||
if (parts.length === 2 && parts[0].length <= 50 && parts[1].length <= 40) {
|
||||
return { company: parts[0], location: parts[1] };
|
||||
}
|
||||
return { company: '', location: '' };
|
||||
}
|
||||
|
||||
// Many posts don't contain the full JD — they link out to the real source
|
||||
// (a career page, hh.ru/geekjob.ru posting, LinkedIn job, etc.), either as a
|
||||
// plain hyperlink in the message text or as an inline-keyboard button
|
||||
// ("Подробнее", "Откликнуться о вакансии"). A t.me/telegram.me href is never
|
||||
// a usable source — it just opens another Telegram entity, and for some
|
||||
// channels (e.g. Runello) that's a bot deep link (t.me/RunelloBot/...) that
|
||||
// only works inside the Telegram client, never a fetchable web page.
|
||||
function isUsableExternalUrl(href) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(href);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') return false;
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
if (host === 't.me' || host === 'telegram.me') return false;
|
||||
// A bare domain root ("http://company.com/") is almost always the
|
||||
// company's marketing homepage, not the specific job posting — some
|
||||
// channels hyperlink the company's NAME to its homepage while the real
|
||||
// apply flow is bot-only (e.g. Runello). A real posting link always has
|
||||
// its own path (/careers/56, /vacancy/123, /jobs/view/123, ...).
|
||||
if (parsed.pathname === '' || parsed.pathname === '/') return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// A recruiter's personal "I'm hiring" LinkedIn post (linkedin.com/posts/...)
|
||||
// is attribution/sharing context, not the job page itself — deprioritize it
|
||||
// below a real posting link (company career page, ATS, linkedin.com/jobs/view/...).
|
||||
function isLinkedinShareUrl(href) {
|
||||
try {
|
||||
const parsed = new URL(href);
|
||||
return /(^|\.)linkedin\.com$/i.test(parsed.hostname) && parsed.pathname.startsWith('/posts/');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Collects every <a href> in document order — both inline links inside the
|
||||
// message text and inline-keyboard buttons share the same tag shape, so one
|
||||
// pass over the whole per-message block catches both. The message's own
|
||||
// permalink/hashtag-search anchors are filtered out by isUsableExternalUrl
|
||||
// (t.me host / relative "?q=" href respectively) without special-casing them.
|
||||
function extractExternalLinks(block) {
|
||||
const hrefs = [];
|
||||
for (const m of block.matchAll(/<a\s[^>]*\bhref="([^"]+)"/gi)) {
|
||||
// Telegram sometimes double-escapes query strings in href attributes
|
||||
// (`&amp;` for a literal `&`) — a second decode pass resolves that;
|
||||
// it's a no-op on already-decoded text since there's no `&` left to match.
|
||||
hrefs.push(decodeHtmlEntities(decodeHtmlEntities(m[1])));
|
||||
}
|
||||
return hrefs.filter(isUsableExternalUrl);
|
||||
}
|
||||
|
||||
// First non-LinkedIn-share candidate wins; a LinkedIn share post is only
|
||||
// used if nothing else was found (it sometimes carries the full JD text
|
||||
// pasted into the post itself).
|
||||
function pickSourceUrl(candidates) {
|
||||
return candidates.find(h => !isLinkedinShareUrl(h)) || candidates[0] || null;
|
||||
}
|
||||
|
||||
// Defensive cap so an unexpected channel format (e.g. a paragraph slipping
|
||||
// past the label/line2 heuristics) can't blow up a pipeline.md/tracker cell.
|
||||
function truncate(s, max) {
|
||||
if (!s || s.length <= max) return s;
|
||||
const cut = s.slice(0, max);
|
||||
const lastSpace = cut.lastIndexOf(' ');
|
||||
return (lastSpace > max * 0.6 ? cut.slice(0, lastSpace) : cut).trim() + '…';
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a t.me/s/{channel} preview page into the canonical Job shape.
|
||||
* Exported for unit tests.
|
||||
*
|
||||
* @param {string} html - raw HTML of the preview page
|
||||
* @param {{channel: string, fallbackCompany?: string}} opts
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, description: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parseTelegramChannelFeed(html, { channel, fallbackCompany } = {}) {
|
||||
if (typeof html !== 'string' || !html) return [];
|
||||
const matches = [...html.matchAll(POST_RE)];
|
||||
const jobs = [];
|
||||
|
||||
for (let i = 0; i < matches.length; i++) {
|
||||
const match = matches[i];
|
||||
const postChannel = match[1];
|
||||
const msgId = match[2];
|
||||
if (channel && postChannel.toLowerCase() !== channel.toLowerCase()) continue;
|
||||
|
||||
const start = match.index ?? 0;
|
||||
const end = i + 1 < matches.length ? (matches[i + 1].index ?? html.length) : html.length;
|
||||
const block = html.slice(start, end);
|
||||
|
||||
const textMatch = block.match(/<div class="tgme_widget_message_text[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
||||
if (!textMatch) continue;
|
||||
const text = htmlBlockToText(textMatch[1]);
|
||||
if (!text) continue;
|
||||
|
||||
const title = extractTitle(text);
|
||||
if (!title || NON_JOB_TITLE_RE.test(title)) continue;
|
||||
|
||||
const labeledCompany = extractField(text, ['компани[яи]', 'работодатель', 'company']);
|
||||
const labeledLocation = extractField(text, ['локаци[яи]', 'город', 'location', 'city']);
|
||||
const line2Guess = (!labeledCompany || !labeledLocation)
|
||||
? extractCompanyLocationFromLine2(text)
|
||||
: { company: '', location: '' };
|
||||
const company = truncate(labeledCompany || line2Guess.company || fallbackCompany || channel, 100);
|
||||
const location = truncate(labeledLocation || line2Guess.location, 60);
|
||||
|
||||
const dateMatch = block.match(/<time[^>]*\bdatetime="([^"]+)"/i);
|
||||
const postedAt = dateMatch ? toEpochMs(dateMatch[1]) : undefined;
|
||||
|
||||
// The bare https://t.me/{channel}/{id} permalink renders only a generic
|
||||
// "Contact @channel" app-install interstitial to a logged-out headless
|
||||
// browser — no message content at all, which trips the liveness
|
||||
// checker's insufficient-content guard and false-flags every post as
|
||||
// expired. The /s/ preview variant (same page family scan() reads)
|
||||
// actually renders the channel content, so it's the fallback of last
|
||||
// resort when the post itself doesn't link out to a real source.
|
||||
const telegramPermalink = `https://t.me/s/${postChannel}/${msgId}`;
|
||||
const sourceUrl = pickSourceUrl(extractExternalLinks(block));
|
||||
|
||||
jobs.push({
|
||||
title,
|
||||
url: sourceUrl || telegramPermalink,
|
||||
company,
|
||||
location,
|
||||
description: text,
|
||||
...(postedAt != null ? { postedAt } : {}),
|
||||
// Traceability back to the originating post once the URL points
|
||||
// elsewhere — pipeline.md renders this as a labeled `| note: ...` cell.
|
||||
...(sourceUrl ? { note: `via Telegram: ${telegramPermalink}` } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'telegram',
|
||||
|
||||
detect(entry) {
|
||||
const channel = channelFromCareersUrl(entry?.careers_url);
|
||||
if (!channel || !CHANNEL_RE.test(channel)) return null;
|
||||
return { url: entry.careers_url };
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const channel = resolveChannel(entry);
|
||||
if (!channel || !CHANNEL_RE.test(channel)) {
|
||||
throw new Error(`telegram: missing or invalid "channel" (public Telegram username) for "${entry?.name || 'entry'}"`);
|
||||
}
|
||||
|
||||
const feedUrl = assertTelegramUrl(`https://t.me/s/${channel}`);
|
||||
const html = await ctx.fetchText(feedUrl, { redirect: 'error' });
|
||||
|
||||
const fallbackCompany = typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : channel;
|
||||
return parseTelegramChannelFeed(html, { channel, fallbackCompany });
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,140 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// The Hub provider — board-wide aggregator feed (Nordic / EU startups):
|
||||
// https://thehub.io/api/jobs
|
||||
// Response shape: { docs: [ { id, key, title, company: { name, ... },
|
||||
// location: { address, locality, country }, absoluteJobUrl, isRemote,
|
||||
// publishedAt, createdAt, ... } ], total, page, pages, limit }
|
||||
//
|
||||
// Paginated 15/page via `?page=N` (1-indexed); the response carries `pages`, so
|
||||
// iteration is bounded by min(pages, max_pages). Default cap is modest (the
|
||||
// board is ~1000 jobs / ~67 pages); override with `max_pages` on the entry.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: thehub`.
|
||||
|
||||
const FEED_BASE = 'https://thehub.io/api/jobs';
|
||||
const TRUSTED_HOST = 'thehub.io';
|
||||
const PER_PAGE = 15;
|
||||
const DEFAULT_MAX_PAGES = 3;
|
||||
const MAX_PAGES_CAP = 67;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertHubUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`thehub: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`thehub: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST) {
|
||||
throw new Error(`thehub: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
|
||||
function resolveMaxPages(entry) {
|
||||
const v = entry?.max_pages;
|
||||
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
|
||||
return DEFAULT_MAX_PAGES;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (typeof value !== 'string' || !value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a single The Hub job. Exported for unit tests.
|
||||
*
|
||||
* Field mapping → the normalized Job shape:
|
||||
* - title: `title`, trimmed (postings without one are dropped).
|
||||
* - url: `absoluteJobUrl` — host-locked to `thehub.io` (The Hub always
|
||||
* serves its canonical posting page there; the per-posting `link`
|
||||
* field is an external apply URL and is NOT used). An off-host or
|
||||
* non-https URL drops the posting. url is the dedup key and is
|
||||
* display-only (written to the pipeline/history, never fetched here).
|
||||
* - company: `company.name`, falling back to the portal entry name, then
|
||||
* "The Hub".
|
||||
* - location: `location.address`, else assembled from `location.locality` /
|
||||
* `location.country`; "Remote" is appended when `isRemote` is true.
|
||||
* - postedAt: `publishedAt` (else `createdAt`) ISO date → epoch ms (omitted
|
||||
* when absent/unparseable).
|
||||
*
|
||||
* @param {any} j
|
||||
* @param {string} [fallbackCompany]
|
||||
* @returns {{ title: string, url: string, company: string, location: string, postedAt?: number } | null}
|
||||
*/
|
||||
export function normalizeHubJob(j, fallbackCompany) {
|
||||
if (!j || typeof j !== 'object') return null;
|
||||
|
||||
const title = typeof j.title === 'string' ? j.title.trim() : '';
|
||||
if (!title) return null;
|
||||
|
||||
let url = '';
|
||||
const rawUrl = typeof j.absoluteJobUrl === 'string' ? j.absoluteJobUrl.trim() : '';
|
||||
if (rawUrl) {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol === 'https:' && parsed.hostname === TRUSTED_HOST) url = parsed.href;
|
||||
} catch {
|
||||
// malformed URL → leave url = '' → dropped below
|
||||
}
|
||||
}
|
||||
if (!url) return null;
|
||||
|
||||
const company =
|
||||
j.company && typeof j.company === 'object' && typeof j.company.name === 'string' && j.company.name.trim()
|
||||
? j.company.name.trim()
|
||||
: typeof fallbackCompany === 'string' && fallbackCompany.trim()
|
||||
? fallbackCompany.trim()
|
||||
: 'The Hub';
|
||||
|
||||
const loc = j.location && typeof j.location === 'object' ? j.location : {};
|
||||
const address = typeof loc.address === 'string' ? loc.address.trim() : '';
|
||||
const locality = typeof loc.locality === 'string' ? loc.locality.trim() : '';
|
||||
const country = typeof loc.country === 'string' ? loc.country.trim() : '';
|
||||
const base = address || [locality, country].filter(Boolean).join(', ');
|
||||
const location = [base, j.isRemote === true ? 'Remote' : ''].filter(Boolean).join(', ');
|
||||
|
||||
/** @type {{ title: string, url: string, company: string, location: string, postedAt?: number }} */
|
||||
const job = { title, url, company, location };
|
||||
const postedAt = toEpochMs(j.publishedAt) ?? toEpochMs(j.createdAt);
|
||||
if (postedAt !== undefined) job.postedAt = postedAt;
|
||||
return job;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'thehub',
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
assertHubUrl(FEED_BASE);
|
||||
const maxPages = resolveMaxPages(entry);
|
||||
const fallbackCompany = entry?.name;
|
||||
const out = [];
|
||||
|
||||
for (let page = 1; page <= maxPages; page++) {
|
||||
const url = `${FEED_BASE}?page=${page}`;
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(url, { redirect: 'error' });
|
||||
if (!json || !Array.isArray(json.docs)) {
|
||||
throw new Error(
|
||||
`thehub: unexpected API response on page ${page} — expected { docs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`,
|
||||
);
|
||||
}
|
||||
for (const j of json.docs) {
|
||||
const normalized = normalizeHubJob(j, fallbackCompany);
|
||||
if (normalized) out.push(normalized);
|
||||
}
|
||||
// Stop at the last page: a short page, or page >= the reported total pages.
|
||||
if (json.docs.length < PER_PAGE) break;
|
||||
if (Number.isInteger(json.pages) && page >= json.pages) break;
|
||||
}
|
||||
return out;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,85 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// The Muse provider — public, zero-auth JSON jobs feed.
|
||||
// Endpoint: https://www.themuse.com/api/public/jobs?page={n}
|
||||
// Response shape: { results: [...], page: n, page_count: N }
|
||||
// All pages are fetched sequentially and aggregated before normalizing.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: themuse`.
|
||||
|
||||
const FEED_BASE = 'https://www.themuse.com/api/public/jobs';
|
||||
const TRUSTED_HOST = 'www.themuse.com';
|
||||
|
||||
/** @param {string} url */
|
||||
function assertMuseUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`themuse: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`themuse: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST) {
|
||||
throw new Error(`themuse: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a single result from the Muse API response. Exported for unit tests.
|
||||
*
|
||||
* Field mapping:
|
||||
* name → title
|
||||
* refs.landing_page → url
|
||||
* company.name → company
|
||||
* locations[0].name → location
|
||||
*
|
||||
* Returns null when required fields (title or url) are missing or invalid.
|
||||
*
|
||||
* @param {any} j
|
||||
* @returns {{ title: string, url: string, company: string, location: string } | null}
|
||||
*/
|
||||
export function normalizeMuseJob(j) {
|
||||
if (!j || typeof j !== 'object') return null;
|
||||
const title = typeof j.name === 'string' ? j.name.trim() : '';
|
||||
if (!title) return null;
|
||||
const url = typeof j.refs?.landing_page === 'string' ? j.refs.landing_page.trim() : '';
|
||||
if (!url || !/^https?:\/\//i.test(url)) return null;
|
||||
const company =
|
||||
typeof j.company?.name === 'string' && j.company.name.trim()
|
||||
? j.company.name.trim()
|
||||
: 'The Muse';
|
||||
const location =
|
||||
Array.isArray(j.locations) && j.locations.length > 0 && typeof j.locations[0]?.name === 'string'
|
||||
? j.locations[0].name.trim()
|
||||
: '';
|
||||
return { title, url, company, location };
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'themuse',
|
||||
|
||||
async fetch(_entry, ctx) {
|
||||
assertMuseUrl(FEED_BASE);
|
||||
const allResults = [];
|
||||
// Fetch page 0 first to discover page_count, then iterate remaining pages.
|
||||
let pageCount = 1;
|
||||
for (let page = 0; page < pageCount; page++) {
|
||||
const url = `${FEED_BASE}?page=${page}`;
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const json = await ctx.fetchJson(url, { redirect: 'error' });
|
||||
if (!json || !Array.isArray(json.results)) {
|
||||
throw new Error(
|
||||
`themuse: unexpected API response on page ${page} — expected { results: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`,
|
||||
);
|
||||
}
|
||||
if (page === 0 && Number.isInteger(json.page_count) && json.page_count > 1) {
|
||||
pageCount = Math.min(json.page_count, 100);
|
||||
}
|
||||
allResults.push(...json.results);
|
||||
}
|
||||
return allResults.map(normalizeMuseJob).filter(Boolean);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,144 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// "Работа России" (trudvsem.ru) provider — hits the Russian government's
|
||||
// public open-data vacancy API (opendata.trudvsem.ru). Aggregates postings
|
||||
// submitted to the state employment service; no auth/API key required.
|
||||
//
|
||||
// Docs: https://trudvsem.ru/opendata/api
|
||||
//
|
||||
// Portal entry fields (all optional except `provider`):
|
||||
// searchKeywords — `text` query param, e.g. "Frontend Vue" (default: "")
|
||||
// region — region code for the `region` path filter (default: none — all regions)
|
||||
// pageSize — `limit` per page, API max is 100 (default: 100)
|
||||
// maxPages — maximum pages to fetch via `offset` (default: 3)
|
||||
|
||||
const API_HOST = 'opendata.trudvsem.ru';
|
||||
const API_BASE = 'https://opendata.trudvsem.ru/api/v1/vacancies';
|
||||
|
||||
const DEFAULT_PAGE_SIZE = 100;
|
||||
const DEFAULT_MAX_PAGES = 3;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertTrudvsemUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`trudvsem: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`trudvsem: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== API_HOST)
|
||||
throw new Error(`trudvsem: untrusted hostname "${parsed.hostname}" — must be ${API_HOST}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single trudvsem API result item into the canonical Job shape.
|
||||
*
|
||||
* The API returns results shaped like:
|
||||
* { results: { vacancies: [ { vacancy: {
|
||||
* "job-name": "...", vac_url: "https://trudvsem.ru/vacancy/card/...",
|
||||
* company: { name: "..." }, region: { name: "..." },
|
||||
* "creation-date": "2026-07-01", date_modify: "2026-07-02T14:32:05+0300"
|
||||
* } } ] } }
|
||||
*
|
||||
* @param {any} item — raw `{ vacancy: {...} }` wrapper from the results array
|
||||
* @param {string} fallbackCompany — company name fallback from the portal entry
|
||||
* @returns {{title: string, url: string, company: string, location: string, postedAt: number|undefined}|null}
|
||||
*/
|
||||
export function parseTrudvsemItem(item, fallbackCompany) {
|
||||
const vacancy = item?.vacancy;
|
||||
if (!vacancy || typeof vacancy !== 'object') return null;
|
||||
|
||||
const title = (vacancy['job-name'] || '').trim();
|
||||
if (!title) return null;
|
||||
|
||||
const rawUrl = (vacancy.vac_url || '').trim();
|
||||
if (!rawUrl) return null;
|
||||
let url;
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.protocol !== 'https:') return null;
|
||||
url = parsed.href;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const company = (vacancy.company?.name || fallbackCompany || '').trim();
|
||||
const location = (vacancy.region?.name || '').trim();
|
||||
const postedAt = toEpochMs(vacancy.date_modify || vacancy['creation-date']);
|
||||
|
||||
return { title, url, company, location, ...(postedAt != null ? { postedAt } : {}) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the search URL with query parameters.
|
||||
* @param {{keywords?: string, region?: string, limit?: number, offset?: number}} params
|
||||
* @returns {string}
|
||||
*/
|
||||
function buildSearchUrl({ keywords, region, limit, offset }) {
|
||||
const base = region ? `${API_BASE}/region/${encodeURIComponent(region)}` : API_BASE;
|
||||
const url = new URL(base);
|
||||
if (keywords) url.searchParams.set('text', keywords);
|
||||
url.searchParams.set('limit', String(limit || DEFAULT_PAGE_SIZE));
|
||||
url.searchParams.set('offset', String(offset || 0));
|
||||
return url.href;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'trudvsem',
|
||||
|
||||
detect(_entry) {
|
||||
// trudvsem is a job board aggregator, not a company ATS.
|
||||
// Use `provider: trudvsem` explicitly in portals.yml.
|
||||
return null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const keywords = entry.searchKeywords || '';
|
||||
const region = entry.region ? String(entry.region) : '';
|
||||
const pageSize = Number(entry.pageSize) || DEFAULT_PAGE_SIZE;
|
||||
const maxPages = Number(entry.maxPages) || DEFAULT_MAX_PAGES;
|
||||
const fallbackCompany = entry.name || '';
|
||||
|
||||
const allJobs = [];
|
||||
|
||||
for (let page = 0; page < maxPages; page++) {
|
||||
const offset = page * pageSize;
|
||||
const searchUrl = buildSearchUrl({ keywords, region, limit: pageSize, offset });
|
||||
assertTrudvsemUrl(searchUrl);
|
||||
|
||||
let json;
|
||||
try {
|
||||
json = /** @type {any} */ (await ctx.fetchJson(searchUrl, { redirect: 'error' }));
|
||||
} catch (err) {
|
||||
if (page === 0) throw err;
|
||||
console.error(`trudvsem: page ${page + 1} fetch failed — ${err.message}`);
|
||||
break;
|
||||
}
|
||||
|
||||
const vacancies = Array.isArray(json?.results?.vacancies) ? json.results.vacancies : [];
|
||||
if (vacancies.length === 0) break;
|
||||
|
||||
for (const item of vacancies) {
|
||||
const job = parseTrudvsemItem(item, fallbackCompany);
|
||||
if (job) allJobs.push(job);
|
||||
}
|
||||
|
||||
if (vacancies.length < pageSize) break;
|
||||
|
||||
// Be a polite citizen of a free government API
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
}
|
||||
|
||||
return allJobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,155 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// We Work Remotely provider - board-wide RSS feed
|
||||
// (https://weworkremotely.com/remote-jobs.rss). The feed is public, no-auth,
|
||||
// and XML, so it is parsed in-process with the same tiny tag extractor approach
|
||||
// as providers/personio.mjs rather than adding an XML dependency.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: weworkremotely`.
|
||||
|
||||
const FEED_URL = 'https://weworkremotely.com/remote-jobs.rss';
|
||||
const TRUSTED_HOST = 'weworkremotely.com';
|
||||
|
||||
/** @param {string} url */
|
||||
function assertWwrUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`weworkremotely: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`weworkremotely: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST) {
|
||||
throw new Error(`weworkremotely: untrusted hostname "${parsed.hostname}" - must be ${TRUSTED_HOST}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
function fallbackCompany(entry) {
|
||||
return typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : 'We Work Remotely';
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'weworkremotely',
|
||||
|
||||
detect(entry) {
|
||||
return entry?.provider === 'weworkremotely' ? { url: FEED_URL } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const feedUrl = assertWwrUrl(FEED_URL);
|
||||
// redirect:'error' prevents SSRF via server-side redirects; combined with
|
||||
// assertWwrUrl above it keeps the request pinned to weworkremotely.com.
|
||||
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
|
||||
return parseWwrFeed(text, fallbackCompany(entry));
|
||||
},
|
||||
};
|
||||
|
||||
function fromCodePoint(cp) {
|
||||
try {
|
||||
return String.fromCodePoint(cp);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// Decode the XML entities that appear in RSS text: numeric (& / ')
|
||||
// and the named five. Numeric forms are decoded first; & is decoded LAST
|
||||
// so a literal "&lt;" yields "<" rather than over-decoding to "<".
|
||||
function decodeXmlEntities(s) {
|
||||
return s
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
|
||||
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
|
||||
function extractText(inner) {
|
||||
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
|
||||
if (cdata) return cdata[1].trim();
|
||||
return decodeXmlEntities(inner).trim();
|
||||
}
|
||||
|
||||
// Extract the text of the first <tag>...</tag> in a block. Returns '' when absent.
|
||||
function tagText(block, tag) {
|
||||
const m = block.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, 'i'));
|
||||
return m ? extractText(m[1]) : '';
|
||||
}
|
||||
|
||||
function cleanUrl(value) {
|
||||
if (!value) return '';
|
||||
const trimmed = value.trim();
|
||||
try {
|
||||
const parsed = new URL(trimmed);
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
const trusted = host === TRUSTED_HOST || host.endsWith(`.${TRUSTED_HOST}`);
|
||||
return parsed.protocol === 'https:' && trusted ? parsed.href : '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function splitTitle(rawTitle, defaultCompany) {
|
||||
const text = rawTitle.trim();
|
||||
const colon = text.indexOf(':');
|
||||
if (colon > 0) {
|
||||
const company = text.slice(0, colon).trim();
|
||||
const title = text.slice(colon + 1).trim();
|
||||
if (company && title) return { company, title };
|
||||
}
|
||||
return { company: defaultCompany, title: text };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse We Work Remotely's public RSS jobs feed. Exported for unit tests.
|
||||
*
|
||||
* Shape: `<rss><channel><item>...</item>...</channel></rss>`, each item
|
||||
* carrying `<title>`, `<link>`, `<pubDate>`, and often `<region>` plus
|
||||
* `<category>`. The title is usually "Company: Role"; when that split is
|
||||
* present, company and title are normalized separately. The RSS `<link>` is
|
||||
* the dedup key; items without a usable absolute URL are dropped.
|
||||
*
|
||||
* @param {string} xml - raw RSS feed body
|
||||
* @param {string} [defaultCompany] - fallback company for unsplittable titles
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parseWwrFeed(xml, defaultCompany = 'We Work Remotely') {
|
||||
if (typeof xml !== 'string') return [];
|
||||
const fallback = typeof defaultCompany === 'string' && defaultCompany.trim() ? defaultCompany.trim() : 'We Work Remotely';
|
||||
const jobs = [];
|
||||
const blocks = xml.match(/<item\b[^>]*>[\s\S]*?<\/item>/gi) || [];
|
||||
|
||||
for (const item of blocks) {
|
||||
const url = cleanUrl(tagText(item, 'link'));
|
||||
if (!url) continue;
|
||||
|
||||
const rawTitle = tagText(item, 'title');
|
||||
if (!rawTitle) continue;
|
||||
|
||||
const { company, title } = splitTitle(rawTitle, fallback);
|
||||
const location = tagText(item, 'region') || tagText(item, 'category');
|
||||
|
||||
jobs.push({
|
||||
title,
|
||||
company,
|
||||
location,
|
||||
url,
|
||||
postedAt: toEpochMs(tagText(item, 'pubDate')),
|
||||
});
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Workable provider — hits the public markdown feed at /<slug>/jobs.md.
|
||||
// Workable's documented JSON API requires an auth token; the markdown feed
|
||||
// is the only no-auth public surface. Auto-detects from careers_url pattern
|
||||
// `https://apply.workable.com/<slug>`. A tracked_companies entry can also
|
||||
// set `provider: workable` explicitly to bypass detection.
|
||||
|
||||
const ALLOWED_WORKABLE_HOSTS = new Set(['apply.workable.com']);
|
||||
|
||||
function assertWorkableUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`workable: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`workable: URL must use HTTPS: ${url}`);
|
||||
if (!ALLOWED_WORKABLE_HOSTS.has(parsed.hostname)) {
|
||||
throw new Error(`workable: untrusted hostname "${parsed.hostname}" — must be one of: ${[...ALLOWED_WORKABLE_HOSTS].join(', ')}`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
function resolveFeedUrl(entry) {
|
||||
const raw = typeof entry.careers_url === 'string' ? entry.careers_url : '';
|
||||
if (!raw) return null;
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (parsed.protocol !== 'https:') return null;
|
||||
if (parsed.hostname !== 'apply.workable.com') return null;
|
||||
const slug = parsed.pathname.split('/').filter(Boolean)[0];
|
||||
if (!slug) return null;
|
||||
return `https://apply.workable.com/${slug}/jobs.md`;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'workable',
|
||||
|
||||
detect(entry) {
|
||||
const feedUrl = resolveFeedUrl(entry);
|
||||
return feedUrl ? { url: feedUrl } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const feedUrl = resolveFeedUrl(entry);
|
||||
if (!feedUrl) throw new Error(`workable: cannot derive feed URL for ${entry.name}`);
|
||||
assertWorkableUrl(feedUrl);
|
||||
// redirect:'error' prevents SSRF via server-side redirects; combined with
|
||||
// assertWorkableUrl above it guarantees the final hostname stays in the allowlist.
|
||||
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
|
||||
return parseWorkableMarkdown(text, entry.name);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse Workable's public markdown feed. Exported as a named export for unit
|
||||
* tests. The feed exposes a table:
|
||||
* | Title | Department | Location | Type | Salary | Posted | Details |
|
||||
* where `Details` holds a markdown link
|
||||
* [View](https://apply.workable.com/<slug>/jobs/view/<id>.md)
|
||||
* URLs are validated against `https://apply.workable.com/` — off-domain or
|
||||
* non-HTTPS [View] links are skipped (not emitted).
|
||||
*
|
||||
* @param {string} text — markdown body
|
||||
* @param {string} companyName — value to write into job.company
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string}>}
|
||||
*/
|
||||
export function parseWorkableMarkdown(text, companyName) {
|
||||
if (typeof text !== 'string') return [];
|
||||
const jobs = [];
|
||||
for (const line of text.split('\n')) {
|
||||
if (!line.startsWith('|') || !line.includes('[View]')) continue;
|
||||
const cols = line.split('|').map(c => c.trim());
|
||||
// Cols: ['', title, dept, location, type, salary, posted, '[View](url.md)', '']
|
||||
if (cols.length < 8) continue;
|
||||
const title = cols[1];
|
||||
if (!title || title === 'Title') continue;
|
||||
const location = cols[3] || '';
|
||||
const urlMatch = line.match(/\[View\]\(([^)]+)\)/);
|
||||
let url = urlMatch ? urlMatch[1] : '';
|
||||
if (url.endsWith('.md')) url = url.slice(0, -3);
|
||||
if (!url) continue; // skip rows with no resolvable URL (e.g., malformed [View] link)
|
||||
|
||||
// Validate the extracted URL — must parse as https://apply.workable.com/...
|
||||
try {
|
||||
const parsedUrl = new URL(url);
|
||||
if (parsedUrl.protocol !== 'https:' || parsedUrl.hostname !== 'apply.workable.com') continue;
|
||||
url = parsedUrl.href;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
jobs.push({ title, url, location, company: companyName });
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Workday provider — hits the public CXS jobs endpoint (POST, paginated).
|
||||
// Auto-detects from careers_url pattern
|
||||
// `https://<tenant>.<instance>.myworkdayjobs.com[/<locale>]/<site>`,
|
||||
// e.g. https://23andme.wd5.myworkdayjobs.com/23 →
|
||||
// POST https://23andme.wd5.myworkdayjobs.com/wday/cxs/23andme/23/jobs
|
||||
//
|
||||
// Workday only exposes a relative "postedOn" label ("Posted Today",
|
||||
// "Posted 5 Days Ago", "Posted 30+ Days Ago"); postedAt is derived from it
|
||||
// and omitted for the unbounded "30+ Days Ago" form.
|
||||
|
||||
const PAGE_SIZE = 20;
|
||||
|
||||
// Safety cap on pagination — applied regardless of what the upstream reports
|
||||
// as `total` (or, when `total` is absent, regardless of how many full pages
|
||||
// keep coming back), so a misbehaving/compromised API can't drive this into
|
||||
// fetching an unbounded number of pages. Override with `max_pages` on the
|
||||
// portal entry for a tenant that genuinely exceeds it.
|
||||
const DEFAULT_MAX_PAGES = 100;
|
||||
// Hard ceiling even for an explicit override. 1500 pages (30,000 postings)
|
||||
// covers known large tenants (dollartree: 23,609; oreillyauto: 17,061;
|
||||
// cvshealth: ~16,800) with headroom — not a completeness guarantee, since a
|
||||
// company directory this size has no fixed upper bound.
|
||||
const MAX_PAGES_CAP = 1500;
|
||||
|
||||
// Retry policy for transient page failures (429 rate-limit, 5xx, timeouts/aborts).
|
||||
// Workday's CXS API is fronted by a WAF that rate-limits in bursts; without
|
||||
// retry, a single 429 silently truncates an entire tenant (e.g. a
|
||||
// 3,383-posting tenant reduced to 20 jobs on page 2). Non-transient errors
|
||||
// (4xx other than 429) are not retried — retrying a malformed request just
|
||||
// wastes the budget.
|
||||
const MAX_RETRIES = 3;
|
||||
const RETRY_BASE_DELAY_MS = 500;
|
||||
const RETRY_MAX_DELAY_MS = 8_000;
|
||||
|
||||
// Delay between successive pages *within one tenant's own pagination loop*
|
||||
// (not between tenants — that's scan-ats-full.mjs's concurrency, a separate
|
||||
// knob). A burst of same-host requests with zero delay risks Workday's
|
||||
// WAF-level rate limiting on any tenant that paginates several pages deep
|
||||
// (large boards like rollsroyce, sec, roche). Only tenants that loop past
|
||||
// page 1 pay this; no-date-skip and early-stopped tenants never do.
|
||||
const INTER_PAGE_DELAY_MS = 150;
|
||||
|
||||
// Workday returns postings newest-first, so pagination can stop once a
|
||||
// page's oldest *dated* posting is well past --since — no point paying for
|
||||
// (and rate-limit-risking) pages that are entirely stale. Only unambiguous
|
||||
// numeric ages ("Posted N Days Ago", N < 30) count for this; the unbounded
|
||||
// "30+ Days Ago" bucket never triggers it, so a wide --since (>=30 days)
|
||||
// simply never early-stops rather than risk a false stop.
|
||||
//
|
||||
// The sort isn't perfectly monotonic day-to-day — some tenants (e.g. Adobe)
|
||||
// return day-labels slightly out of order across consecutive postings ("27
|
||||
// Days Ago | 26 Days Ago | 27 Days Ago"), roughly 1 day of jitter. The
|
||||
// margin only needs to clear that; 2 is double it as a plain safety factor,
|
||||
// not a second measurement.
|
||||
const EARLY_STOP_MARGIN_MS = 2 * 86_400_000;
|
||||
|
||||
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
|
||||
function resolveMaxPages(entry) {
|
||||
const v = entry?.max_pages;
|
||||
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
|
||||
return DEFAULT_MAX_PAGES;
|
||||
}
|
||||
|
||||
function sleep(ms, ctx) {
|
||||
if (typeof ctx?.sleep === 'function') return ctx.sleep(ms);
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Parses a `Retry-After` header value (seconds, or an HTTP-date) to ms, or null. */
|
||||
function parseRetryAfterMs(value) {
|
||||
if (!value) return null;
|
||||
const secs = Number(value);
|
||||
if (Number.isFinite(secs) && secs >= 0) return secs * 1000;
|
||||
const dateMs = Date.parse(value);
|
||||
return Number.isFinite(dateMs) ? Math.max(0, dateMs - Date.now()) : null;
|
||||
}
|
||||
|
||||
function isRetryableError(err) {
|
||||
const status = err?.status;
|
||||
if (status === 429) return true;
|
||||
if (typeof status === 'number' && status >= 500) return true;
|
||||
return status === undefined; // network error / timeout / abort — no status set
|
||||
}
|
||||
|
||||
/** Fetches a single page, retrying transient failures with backoff. */
|
||||
async function fetchPageWithRetry(ctx, api, opts) {
|
||||
let lastErr;
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
return await ctx.fetchJson(api, opts);
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
if (attempt === MAX_RETRIES || !isRetryableError(err)) throw err;
|
||||
const backoff = Math.min(RETRY_BASE_DELAY_MS * 2 ** attempt, RETRY_MAX_DELAY_MS);
|
||||
// A server-supplied Retry-After is honored, but still clamped — an
|
||||
// unbounded value (hostile or just misconfigured: Retry-After: 86400)
|
||||
// would otherwise stall this tenant's fetch for as long as the server
|
||||
// says, defeating the whole point of a bounded backoff.
|
||||
const retryAfterMs = parseRetryAfterMs(err?.retryAfter);
|
||||
const delayMs = retryAfterMs !== null ? Math.min(retryAfterMs, RETRY_MAX_DELAY_MS * 4) : (backoff + Math.random() * 250);
|
||||
await sleep(delayMs, ctx);
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
/** True once a page's oldest unambiguously-dated posting is past the --since window. */
|
||||
function pageIsPastWindow(pageJobs, sinceMs) {
|
||||
if (typeof sinceMs !== 'number') return false;
|
||||
const dated = pageJobs.map((j) => j.postedAt).filter((v) => typeof v === 'number');
|
||||
if (dated.length === 0) return false;
|
||||
return Math.min(...dated) < sinceMs - EARLY_STOP_MARGIN_MS;
|
||||
}
|
||||
|
||||
function resolveEndpoint(entry) {
|
||||
// Try api: first, then careers_url (mirrors greenhouse/ashby), returning the
|
||||
// first that matches the Workday tenant pattern. This lets a branded page
|
||||
// (e.g. https://www.ptc.com/en/careers) stay as careers_url while the Workday
|
||||
// tenant URL is pinned via api: — and, because we fall through on a non-match,
|
||||
// a non-Workday api: value doesn't shadow a valid careers_url.
|
||||
for (const url of [entry.api, entry.careers_url]) {
|
||||
if (typeof url !== 'string' || !url) continue;
|
||||
const m = url.match(/^https:\/\/([\w-]+)\.(wd[\w-]*)\.myworkdayjobs\.com\/(?:[a-z]{2}-[A-Z]{2}\/)?([^/?#]+)/);
|
||||
if (!m) continue;
|
||||
const [, tenant, instance, site] = m;
|
||||
const origin = `https://${tenant}.${instance}.myworkdayjobs.com`;
|
||||
return {
|
||||
api: `${origin}/wday/cxs/${tenant}/${site}/jobs`,
|
||||
// externalPath is relative to the site, not the host root — without the
|
||||
// site segment the URL 404s.
|
||||
jobBase: `${origin}/${site}`,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parsePostedOn(label) {
|
||||
if (!label) return undefined;
|
||||
if (/posted\s+today/i.test(label)) return Date.now();
|
||||
if (/posted\s+yesterday/i.test(label)) return Date.now() - 86_400_000;
|
||||
const m = label.match(/posted\s+(\d+)(\+?)\s*day/i);
|
||||
if (!m || m[2] === '+') return undefined; // "30+ Days Ago" — unbounded, no usable date
|
||||
return Date.now() - Number(m[1]) * 86_400_000;
|
||||
}
|
||||
|
||||
// Workday URL path encodes location as /job/{Location-Slug}/{title-slug}.
|
||||
// Use it as fallback when locationsText is absent (common on some tenants).
|
||||
function locationFromPath(externalPath) {
|
||||
const m = String(externalPath || '').match(/\/job\/([^/]+)\//);
|
||||
if (!m) return '';
|
||||
let segment;
|
||||
try { segment = decodeURIComponent(m[1]); } catch { segment = m[1]; }
|
||||
return segment.replace(/-/g, ' ');
|
||||
}
|
||||
|
||||
export function parseWorkdayResponse(json, entry) {
|
||||
const ep = resolveEndpoint(entry);
|
||||
const jobBase = ep?.jobBase || '';
|
||||
const postings = Array.isArray(json?.jobPostings) ? json.jobPostings : [];
|
||||
const jobs = [];
|
||||
for (const j of postings) {
|
||||
if (j == null) continue;
|
||||
if (!j.externalPath || !String(j.title || '').trim()) continue;
|
||||
jobs.push({
|
||||
title: j.title || '',
|
||||
url: jobBase + j.externalPath,
|
||||
company: entry.name,
|
||||
location: j.locationsText || locationFromPath(j.externalPath),
|
||||
postedAt: parsePostedOn(j.postedOn),
|
||||
});
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'workday',
|
||||
|
||||
detect(entry) {
|
||||
const ep = resolveEndpoint(entry);
|
||||
return ep ? { url: ep.api } : null;
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const ep = resolveEndpoint(entry);
|
||||
if (!ep) throw new Error(`workday: cannot derive CXS endpoint for ${entry.name}`);
|
||||
|
||||
const postOpts = { method: 'POST', redirect: 'error', headers: { 'content-type': 'application/json', accept: 'application/json' } };
|
||||
const makeBody = (offset) => JSON.stringify({ limit: PAGE_SIZE, offset, searchText: '', appliedFacets: {} });
|
||||
const sinceMs = typeof ctx?.sinceMs === 'number' ? ctx.sinceMs : null;
|
||||
|
||||
const first = await fetchPageWithRetry(ctx, ep.api, { ...postOpts, body: makeBody(0) });
|
||||
const jobs = parseWorkdayResponse(first, entry);
|
||||
|
||||
const total = typeof first?.total === 'number' ? first.total : null;
|
||||
const firstPostings = Array.isArray(first?.jobPostings) ? first.jobPostings : [];
|
||||
const maxPages = resolveMaxPages(entry);
|
||||
|
||||
// How many pages to fetch in total (including the first, already-fetched
|
||||
// one): bounded by `total` when the server reports it, always capped at
|
||||
// maxPages. When `total` is absent, only probe further pages if the first
|
||||
// one was full — a short first page already means there's nothing more.
|
||||
let pagesToFetch = total !== null
|
||||
? Math.min(Math.ceil(total / PAGE_SIZE), maxPages)
|
||||
: (firstPostings.length >= PAGE_SIZE ? maxPages : 1);
|
||||
|
||||
// Honor a context page cap — verify-portals' liveness probe sets
|
||||
// `ctx.maxPages: 1` so it only needs to know a board is live, not its full
|
||||
// count. Without this we'd fetch page 0, then request page 1 and trip the
|
||||
// probe's second-request sentinel; fetchPageWithRetry treats that abort as
|
||||
// transient and retries it MAX_RETRIES times (with backoff) before giving up
|
||||
// — noisy in the logs and rude to the tenant. Capping here makes workday a
|
||||
// "cooperating provider" that stops after one page and reports an exact
|
||||
// first-page count. Kept separate from `maxPages` so the entry-cap warning
|
||||
// below (pagesToFetch === maxPages) stays quiet. No effect on real scans,
|
||||
// which don't set ctx.maxPages.
|
||||
const ctxCap = Number.isInteger(ctx?.maxPages) && ctx.maxPages > 0 ? ctx.maxPages : Infinity;
|
||||
pagesToFetch = Math.min(pagesToFetch, ctxCap);
|
||||
|
||||
// Why pagination stopped — drives which warning (if any) fires below.
|
||||
// 'fetch-error' must NOT produce the "raise max_pages" advice: that knob
|
||||
// does nothing for a tenant that died on a rate limit rather than hit the cap.
|
||||
let stopReason = 'complete';
|
||||
if (pageIsPastWindow(jobs, sinceMs)) stopReason = 'early-stop';
|
||||
// Some tenants' CXS responses never include postedOn at all (e.g.
|
||||
// adventhealth, on every page). Early-stop can't apply then — there's
|
||||
// no dated posting to recognize as "past the window".
|
||||
const sawAnyDatedPosting = jobs.some((j) => typeof j.postedAt === 'number');
|
||||
|
||||
// Zero dated postings on page 0, --include-undated off, --since-bounded
|
||||
// scan: further pagination is pure waste — every posting from this
|
||||
// tenant will be dropped downstream as undated regardless of page count
|
||||
// (newest-first sort means if the *freshest* postings lack a date, older
|
||||
// ones will too). Return page 0's results instead of grinding to maxPages.
|
||||
if (stopReason === 'complete' && sinceMs !== null && ctx?.includeUndated !== true
|
||||
&& !sawAnyDatedPosting && jobs.length > 0) {
|
||||
stopReason = 'no-date-skip';
|
||||
}
|
||||
|
||||
// Sequential, not concurrent (mirrors providers/4dayweek.mjs, thehub.mjs,
|
||||
// arbeitnow.mjs, jibeapply.mjs) — a single tenant's API has no reason to
|
||||
// receive a burst of parallel requests, and a mid-run failure stops
|
||||
// cleanly with whatever pages were already gathered instead of
|
||||
// discarding them (Promise.all would fail the whole batch on one error).
|
||||
let page = 1;
|
||||
if (stopReason === 'complete') {
|
||||
for (; page < pagesToFetch; page++) {
|
||||
await sleep(INTER_PAGE_DELAY_MS, ctx);
|
||||
let json;
|
||||
try {
|
||||
json = await fetchPageWithRetry(ctx, ep.api, { ...postOpts, body: makeBody(page * PAGE_SIZE) });
|
||||
} catch (err) {
|
||||
const jobsSummary = `${jobs.length}${total !== null ? ` of ${total}` : ''} jobs`;
|
||||
console.error(`⚠️ workday: ${entry.name} truncated at ${page + 1} of ${pagesToFetch} pages after ${MAX_RETRIES + 1} attempts (${jobsSummary}): ${err.message}`);
|
||||
stopReason = 'fetch-error';
|
||||
break;
|
||||
}
|
||||
const pageJobs = parseWorkdayResponse(json, entry);
|
||||
jobs.push(...pageJobs);
|
||||
if (total === null) {
|
||||
const postings = Array.isArray(json?.jobPostings) ? json.jobPostings : [];
|
||||
if (postings.length < PAGE_SIZE) break; // short page → last page reached
|
||||
}
|
||||
if (pageIsPastWindow(pageJobs, sinceMs)) { stopReason = 'early-stop'; break; }
|
||||
}
|
||||
if (stopReason === 'complete' && page === pagesToFetch && pagesToFetch === maxPages) {
|
||||
stopReason = 'cap';
|
||||
}
|
||||
}
|
||||
|
||||
// The cap is a safety net, not a working limit — silent by design, but a
|
||||
// tenant that actually hits it needs to be surfaced, in one short line
|
||||
// (a full-directory scan can hit this on dozens of tenants).
|
||||
//
|
||||
// "raise max_pages" only applies when `entry` is a real portals.yml
|
||||
// tracked_companies entry (scan.mjs, sinceMs === null). scan-ats-full.mjs's
|
||||
// reverse scan (the only caller that sets ctx.sinceMs) synthesizes entries
|
||||
// from the external dataset — there's no portal entry to edit, and no
|
||||
// fixed cap can guarantee full coverage of an unbounded company
|
||||
// directory anyway, so there's nothing else to suggest.
|
||||
if (stopReason === 'cap') {
|
||||
const jobsSummary = `${jobs.length}${total !== null ? ` of ${total}` : ''} jobs`;
|
||||
if (sinceMs === null) {
|
||||
console.error(`⚠️ workday: ${entry.name} truncated at max_pages=${maxPages} (${jobsSummary}) — raise max_pages on this entry for more`);
|
||||
} else {
|
||||
// Workday's CXS backend can report `total` as exactly
|
||||
// maxPages*PAGE_SIZE when the real count is far higher (e.g.
|
||||
// dickssportinggoods: total=2000, public site lists 7,120; requests
|
||||
// at offset 2000/4000 return the same first posting as offset 0).
|
||||
// Flag it, don't explain it here.
|
||||
const suspectTag = total !== null && total === maxPages * PAGE_SIZE ? ' (total may be Workday-capped, not real)' : '';
|
||||
console.error(`⚠️ workday: ${entry.name} truncated at ${maxPages} pages (${jobsSummary})${suspectTag}`);
|
||||
}
|
||||
}
|
||||
// 'no-date-skip' hits many tenants in a full-directory scan (a company
|
||||
// with several Workday sites, like a1group or ashealthnet, triggers it
|
||||
// once per site) — a console.error per hit would repeat thousands of
|
||||
// times, so tag the array instead; scan-ats-full.mjs aggregates it into
|
||||
// one summary line.
|
||||
if (stopReason === 'no-date-skip') jobs.workdayNoDateSkip = true;
|
||||
|
||||
return jobs;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,40 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Working Nomads provider — board-wide aggregator feed
|
||||
// (https://www.workingnomads.com/api/exposed_jobs/). Returns a JSON array of
|
||||
// postings; scan.mjs applies the configured title_filter / location_filter.
|
||||
//
|
||||
// Wire in via a `job_boards:` entry with `provider: workingnomads`.
|
||||
|
||||
const FEED_URL = 'https://www.workingnomads.com/api/exposed_jobs/';
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'workingnomads',
|
||||
|
||||
/**
|
||||
* Fetches and normalizes postings from the Working Nomads public feed.
|
||||
* @param {{ name?: string }} entry - The job_boards entry being processed.
|
||||
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
|
||||
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>}
|
||||
*/
|
||||
async fetch(entry, ctx) {
|
||||
// redirect:'error' prevents SSRF via server-side redirects
|
||||
const data = await ctx.fetchJson(FEED_URL, { redirect: 'error' });
|
||||
if (!Array.isArray(data)) {
|
||||
throw new Error(`workingnomads: unexpected API response — expected a JSON array, got ${data === null ? 'null' : typeof data}`);
|
||||
}
|
||||
|
||||
return data
|
||||
.filter(j => j && typeof j === 'object'
|
||||
&& typeof j.title === 'string' && j.title.trim() !== ''
|
||||
&& typeof j.url === 'string' && /^https?:\/\//i.test(j.url.trim()))
|
||||
.map(j => ({
|
||||
title: j.title.trim(),
|
||||
url: j.url.trim(),
|
||||
company: typeof j.company_name === 'string' && j.company_name.trim() ? j.company_name.trim() : (entry.name || 'Working Nomads'),
|
||||
location: typeof j.location === 'string' ? j.location.trim() : '',
|
||||
}));
|
||||
},
|
||||
};
|
||||
Reference in new issue
Block a user