init
Release Please / release-please (push) Failing after 4m2s
Web CI / web typecheck + build (push) Successful in 3m39s
CodeQL Analysis / Analyze (javascript-typescript) (push) Failing after 1h1m49s
CodeQL Analysis / Analyze (go) (push) Failing after 1h1m49s

This commit is contained in:
redlineo committed 2026-07-13 10:14:00 +03:00
commit d295c7f139
1527 files changed
+270726

No files matched your search

+137
View File
@@ -0,0 +1,137 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// 4 Day Week provider — board-wide aggregator of 4-day-week / reduced-hours
// roles. Public, zero-auth JSON API: https://4dayweek.io/api/jobs
// Response shape: { jobs: [ { id, title, slug, company_name, company: { name,
// slug, ... }, work_arrangement, locations: [{ city, country, ... }],
// posted (epoch SECONDS), is_expired, ... } ], total, page, has_more }
//
// The feed carries no per-job URL — the canonical posting page is
// https://4dayweek.io/job/<slug> (verified via the page's rel=canonical; the
// /jobs and /remote-jobs paths soft-404 to the listing). The slug is the only
// variable part of the built URL and is constrained to a safe token.
//
// Paginated 25/page via ?page=N; bounded by max_pages (default 3, cap 50) and
// the has_more flag. Expired postings (is_expired) are dropped.
//
// Wire in via a `job_boards:` entry with `provider: 4dayweek`.
const FEED_BASE = 'https://4dayweek.io/api/jobs';
const TRUSTED_HOST = '4dayweek.io';
const JOB_BASE = `https://${TRUSTED_HOST}/job`;
const PER_PAGE = 25;
const DEFAULT_MAX_PAGES = 3;
const MAX_PAGES_CAP = 50;
const SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9-]*$/;
/** @param {string} url */
function assertFourDayUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`4dayweek: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`4dayweek: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST) {
throw new Error(`4dayweek: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
}
return url;
}
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
function resolveMaxPages(entry) {
const v = entry?.max_pages;
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
return DEFAULT_MAX_PAGES;
}
// NaN-safe: posted is epoch SECONDS → ms; anything non-finite yields undefined.
function toEpochMs(seconds) {
return Number.isFinite(seconds) ? seconds * 1000 : undefined;
}
/**
* Normalize a single 4 Day Week job. Exported for unit tests.
*
* Field mapping → the normalized Job shape:
* - title: `title`, trimmed (postings without one are dropped).
* - url: built as `https://4dayweek.io/job/<slug>` — the feed has no URL
* field. The slug must match a safe token (no slashes/spaces) or
* the posting is dropped; the URL is the dedup key and display-only.
* - company: `company_name` (else `company.name`), falling back to the portal
* entry name, then "4 Day Week".
* - location: first `locations[]` as "city, country"; "Remote" is appended when
* the job's (or location's) work_arrangement is "remote".
* - postedAt: `posted` (epoch SECONDS) → epoch ms (omitted when absent).
*
* Expired postings (`is_expired === true`) are dropped.
*
* @param {any} j
* @param {string} [fallbackCompany]
* @returns {{ title: string, url: string, company: string, location: string, postedAt?: number } | null}
*/
export function normalize4dwJob(j, fallbackCompany) {
if (!j || typeof j !== 'object') return null;
if (j.is_expired === true) return null;
const title = typeof j.title === 'string' ? j.title.trim() : '';
if (!title) return null;
const slug = typeof j.slug === 'string' ? j.slug.trim() : '';
if (!SLUG_RE.test(slug)) return null; // need a clean slug to build the url
const url = `${JOB_BASE}/${encodeURIComponent(slug)}`;
const company =
typeof j.company_name === 'string' && j.company_name.trim()
? j.company_name.trim()
: j.company && typeof j.company === 'object' && typeof j.company.name === 'string' && j.company.name.trim()
? j.company.name.trim()
: typeof fallbackCompany === 'string' && fallbackCompany.trim()
? fallbackCompany.trim()
: '4 Day Week';
const first = Array.isArray(j.locations) && j.locations[0] && typeof j.locations[0] === 'object' ? j.locations[0] : {};
const city = typeof first.city === 'string' ? first.city.trim() : '';
const country = typeof first.country === 'string' ? first.country.trim() : '';
const base = [city, country].filter(Boolean).join(', ');
const remote = j.work_arrangement === 'remote' || first.work_arrangement === 'remote';
const location = [base, remote ? 'Remote' : ''].filter(Boolean).join(', ');
/** @type {{ title: string, url: string, company: string, location: string, postedAt?: number }} */
const job = { title, url, company, location };
const postedAt = toEpochMs(j.posted);
if (postedAt !== undefined) job.postedAt = postedAt;
return job;
}
/** @type {Provider} */
export default {
id: '4dayweek',
async fetch(entry, ctx) {
assertFourDayUrl(FEED_BASE);
const maxPages = resolveMaxPages(entry);
const fallbackCompany = entry?.name;
const out = [];
for (let page = 1; page <= maxPages; page++) {
const url = `${FEED_BASE}?page=${page}`;
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(url, { redirect: 'error' });
if (!json || !Array.isArray(json.jobs)) {
throw new Error(
`4dayweek: unexpected API response on page ${page} — expected { jobs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`,
);
}
for (const j of json.jobs) {
const normalized = normalize4dwJob(j, fallbackCompany);
if (normalized) out.push(normalized);
}
if (json.has_more === false) break; // last page per the API flag
if (json.jobs.length < PER_PAGE) break; // short page → last page
}
return out;
},
};
+53
View File
@@ -0,0 +1,53 @@
// HTTP transport helpers shared across providers.
// Files prefixed with _ are never loaded as providers by scan.mjs.
const DEFAULT_TIMEOUT_MS = 10_000;
const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; career-ops/1.3)';
async function fetchWithTimeout(url, { timeoutMs = DEFAULT_TIMEOUT_MS, headers = {}, method = 'GET', body = null, redirect = 'follow' } = {}) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const res = await fetch(url, {
method,
headers: { 'user-agent': DEFAULT_USER_AGENT, ...headers },
body,
redirect,
signal: controller.signal,
});
if (!res.ok) {
const responseText = await res.text().catch(() => '');
// WAF/CDN challenge pages (seen live: Workday 429s) carry no actionable
// text — HTML markup or a generic interstitial message, not worth
// parsing or displaying. The status code and its standard reason
// phrase are what a log line needs; the raw body is still attached as
// err.body for callers that want to inspect it.
const err = new Error(`HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ''}`);
err.status = res.status;
err.body = responseText;
err.retryAfter = res.headers.get('retry-after');
throw err;
}
return res;
} finally {
clearTimeout(timer);
}
}
export async function fetchJson(url, opts = {}) {
const res = await fetchWithTimeout(url, opts);
return await res.json();
}
export async function fetchText(url, opts = {}) {
const res = await fetchWithTimeout(url, opts);
return await res.text();
}
export function makeHttpCtx() {
return {
transport: 'http',
fetchJson,
fetchText,
};
}
+94
View File
@@ -0,0 +1,94 @@
// Provider registry — loading and routing shared by the scanner and the portal
// health check. Files prefixed with _ are never loaded as providers by scan.mjs,
// so this helper module lives safely alongside the provider plugins.
//
// Extracted from scan.mjs (#1451-era inline definitions) so verify-portals.mjs
// can route non-ATS boards through the SAME provider layer the scanner uses,
// without importing scan.mjs itself (which has top-level side effects and would
// form an import cycle via classifyFetchError).
import { existsSync, readdirSync } from 'fs';
import path from 'path';
import { pathToFileURL } from 'url';
/**
* Load every provider plugin in a directory into an id→provider Map.
*
* Alphabetical order so detect() priority is deterministic across machines.
* Malformed modules (wrong shape, duplicate id, import error) are logged and
* skipped, never fatal.
*
* @param {string} dir - Absolute path to the providers directory.
* @returns {Promise<Map<string, object>>}
*/
export async function loadProviders(dir) {
const providers = new Map();
if (!existsSync(dir)) return providers;
const entries = readdirSync(dir)
.filter(f => f.endsWith('.mjs') && !f.startsWith('_'))
.sort();
for (const file of entries) {
const full = path.join(dir, file);
let mod;
try {
mod = await import(pathToFileURL(full).href);
} catch (err) {
console.error(`⚠️ ${file}: failed to load — ${err.message}`);
continue;
}
const p = mod.default;
if (!p || typeof p.fetch !== 'function' || !p.id) {
console.error(`⚠️ ${file}: skipping — default export must be { id, fetch }`);
continue;
}
if (providers.has(p.id)) {
console.error(`⚠️ ${file}: duplicate provider id "${p.id}" — keeping first`);
continue;
}
providers.set(p.id, p);
}
return providers;
}
/**
* Resolve which provider handles a tracked_companies entry.
* 1. Explicit `provider:` field wins (skips detect()).
* 2. local-parser when parser.command + script are configured (before API detect).
* 3. Otherwise each provider's detect() runs in load order; first hit wins.
*
* @param {object} entry - tracked_companies entry.
* @param {Map<string, object>} providers - id→provider Map from loadProviders().
* @param {{skipIds?: string[]}} [opts] - Provider ids to skip (e.g. 'local-parser'
* so a network-only health check never execs a configured local command).
* @returns {{provider: object}|{error: string}|null}
*/
export function resolveProvider(entry, providers, { skipIds = [] } = {}) {
if (entry.provider) {
const p = providers.get(entry.provider);
if (!p) return { error: `unknown provider: ${entry.provider}` };
return { provider: p };
}
const localParser = providers.get('local-parser');
if (localParser && !skipIds.includes('local-parser')) {
try {
const hit = localParser.detect?.(entry);
if (hit) return { provider: localParser };
} catch (err) {
console.error(`⚠️ local-parser: detect() threw for "${entry.name}" — ${err.message}`);
}
}
for (const p of providers.values()) {
if (skipIds.includes(p.id)) continue;
let hit;
try {
hit = p.detect?.(entry);
} catch (err) {
console.error(`⚠️ ${p.id}: detect() threw for "${entry.name}" — ${err.message}`);
continue;
}
if (hit) return { provider: p };
}
return null;
}
+218
View File
@@ -0,0 +1,218 @@
// @ts-check
/**
* _trust-validator.mjs — Lightweight trust validation for scanned job postings.
*
* Enriches each job with trustScore (0-100), trustFlags (string[]), and
* trustLevel ('high' | 'medium' | 'low'). Never drops jobs — flag only.
*
* Underscore prefix keeps scan.mjs's provider loader from importing this
* as a job-board provider.
*
* V1 heuristics:
* 1. URL structure validation
* 2. Missing application URL
* 3. Suspicious domain detection
* 4. Company ↔ domain mismatch (with ATS allowlist)
*/
/** @type {string[]} */
const DEFAULT_SUSPICIOUS_DOMAINS = [
'bit.ly',
'tinyurl.com',
't.co',
'forms.gle',
'goo.gl',
'shorturl.at',
'rebrand.ly',
'cutt.ly',
];
/** @type {string[]} */
const DEFAULT_ATS_ALLOWLIST = [
'greenhouse.io',
'ashbyhq.com',
'lever.co',
'workday.com',
'smartrecruiters.com',
'jobvite.com',
'myworkdayjobs.com',
'recruitee.com',
'workable.com',
'icims.com',
'taleo.net',
'applytojob.com',
'breezy.hr',
'jazz.co',
'bamboohr.com',
'teamtailor.com',
];
/** @type {Record<string, number>} */
const PENALTIES = {
invalid_url: 50,
missing_apply_url: 40,
suspicious_domain: 25,
company_domain_mismatch: 15,
};
/**
* Classify a numeric trust score into a human-readable level.
*
* @param {number} score
* @returns {'high' | 'medium' | 'low'}
*/
export function classifyTrustLevel(score) {
if (score >= 90) return 'high';
if (score >= 60) return 'medium';
return 'low';
}
/**
* Check whether a URL string is well-formed and uses http(s).
*
* @param {string} url
* @returns {{ valid: boolean, flag?: string }}
*/
export function validateUrl(url) {
try {
const parsed = new URL(url);
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
return { valid: false, flag: 'invalid_url' };
}
return { valid: true };
} catch {
return { valid: false, flag: 'invalid_url' };
}
}
/**
* Check whether a hostname matches or is a subdomain of any entry in a
* domain list. E.g. hostname "abc.bit.ly" matches blocklist entry "bit.ly".
*
* @param {string} hostname — lowercased hostname from URL
* @param {string[]} domainList
* @returns {boolean}
*/
export function matchesDomainList(hostname, domainList) {
for (const domain of domainList) {
if (hostname === domain || hostname.endsWith('.' + domain)) {
return true;
}
}
return false;
}
/**
* Heuristic: does the company name plausibly match the URL hostname?
*
* Strips non-alphanumeric chars from the company name, lowercases, and checks
* whether the result (or any word ≥3 chars) appears as a substring of the
* hostname. Returns true when a match is found (= no mismatch flag needed).
*
* @param {string} company
* @param {string} hostname
* @returns {boolean} true if company matches hostname
*/
export function companyMatchesHostname(company, hostname) {
if (!company || !hostname) return true; // can't evaluate → no flag
const normalized = company.toLowerCase().replace(/[^a-z0-9 ]/g, '').trim();
if (!normalized) return true;
// Full slug check (all spaces removed)
const slug = normalized.replace(/\s+/g, '');
if (hostname.includes(slug)) return true;
// Word-level check — any word ≥3 chars present in hostname
const words = normalized.split(/\s+/).filter(w => w.length >= 3);
for (const word of words) {
if (hostname.includes(word)) return true;
}
return false;
}
/**
* Build a trust validator function from portals.yml config.
*
* Usage mirrors buildLocationFilter / buildSalaryFilter in scan.mjs.
*
* @param {object} [config]
* @param {boolean} [config.enabled]
* @param {string[]} [config.suspicious_domains]
* @param {string[]} [config.ats_allowlist]
* @returns {(job: { url?: string, company?: string }) => { score: number, flags: string[], level: 'high' | 'medium' | 'low' }}
*/
export function buildTrustValidator(config) {
// Disabled or absent config → return a no-op that gives every job 100/high
if (!config || config.enabled === false) {
return () => ({ score: 100, flags: [], level: /** @type {const} */ ('high') });
}
const suspiciousDomains = (Array.isArray(config.suspicious_domains)
? config.suspicious_domains
: DEFAULT_SUSPICIOUS_DOMAINS)
.map(d => String(d).toLowerCase().trim())
.filter(Boolean);
const atsAllowlist = (Array.isArray(config.ats_allowlist)
? config.ats_allowlist
: DEFAULT_ATS_ALLOWLIST)
.map(d => String(d).toLowerCase().trim())
.filter(Boolean);
return (job) => {
/** @type {string[]} */
const flags = [];
let score = 100;
const url = typeof job.url === 'string' ? job.url.trim() : '';
// Rule 1 — Missing URL
if (!url) {
flags.push('missing_apply_url');
score -= PENALTIES.missing_apply_url;
// Can't run further URL-based checks
const clamped = Math.max(0, score);
return { score: clamped, flags, level: classifyTrustLevel(clamped) };
}
// Rule 2 — URL structure validation
const urlCheck = validateUrl(url);
if (!urlCheck.valid) {
flags.push('invalid_url');
score -= PENALTIES.invalid_url;
// Can't parse hostname → skip domain checks
const clamped = Math.max(0, score);
return { score: clamped, flags, level: classifyTrustLevel(clamped) };
}
// Parse hostname for domain checks
let hostname = '';
try {
hostname = new URL(url).hostname.toLowerCase();
} catch {
// Already validated above, but guard defensively
const clamped = Math.max(0, score);
return { score: clamped, flags, level: classifyTrustLevel(clamped) };
}
// Rule 3 — Suspicious domain detection
if (matchesDomainList(hostname, suspiciousDomains)) {
flags.push('suspicious_domain');
score -= PENALTIES.suspicious_domain;
}
// Rule 4 — Company ↔ domain mismatch (skip for ATS-hosted URLs)
const company = typeof job.company === 'string' ? job.company.trim() : '';
if (company && !matchesDomainList(hostname, atsAllowlist)) {
if (!companyMatchesHostname(company, hostname)) {
flags.push('company_domain_mismatch');
score -= PENALTIES.company_domain_mismatch;
}
}
score = Math.max(0, Math.min(100, score));
return { score, flags, level: classifyTrustLevel(score) };
};
}
+121
View File
@@ -0,0 +1,121 @@
// Type catalog for the provider plugin contract.
//
// This file is documentation-only — pure JSDoc @typedef annotations. The
// project is plain ESM JavaScript with no build step; provider authors can
// reference these types via `/** @typedef {import('./_types.js').Provider} Provider */`
// at the top of a `// @ts-check`-enabled file to get IDE hints. The runtime
// contract is enforced by scan.mjs (id presence, fetch is a function, fetch
// returns an array), not by these annotations.
//
// Files prefixed with _ are never loaded as providers by scan.mjs.
/**
* Normalized job posting — the unit of currency throughout the scanner.
*
* @typedef {object} Job
* @property {string} title Required, non-empty after trim.
* @property {string} url Required, absolute URL — used as the dedup key.
* @property {string} company May be empty when the source can't expose it
* at the list-page level; populated downstream.
* @property {string} location May be empty.
* @property {string} [description] Job description text, populated ONLY when the
* provider's list payload carries it for free (no
* extra per-job request — the scanner is zero-token).
* Lever supplies it via `descriptionPlain`; most
* providers omit it. Consumed by scan.mjs's
* content_filter; an empty/absent value always
* passes the filter.
* @property {number} [postedAt] Epoch ms when the posting was published.
* Omitted when the source doesn't expose a
* usable date. scan.mjs ignores it; consumers
* like scan-ats-full.mjs use it for recency
* filtering.
* @property {number} [trustScore] 0-100 trust score from _trust-validator.mjs.
* @property {string[]} [trustFlags] Flags raised by trust validation (e.g.
* 'invalid_url', 'suspicious_domain').
* @property {'high'|'medium'|'low'} [trustLevel] Classification derived from
* trustScore.
*/
/**
* Result returned by the trust validator for a single job posting.
*
* @typedef {object} TrustResult
* @property {number} score 0-100, where 100 = fully trusted.
* @property {string[]} flags Flags raised (e.g. 'invalid_url', 'suspicious_domain').
* @property {'high'|'medium'|'low'} level Classification: 90-100 high, 60-89 medium, 0-59 low.
*/
/**
* A single `tracked_companies` entry from `portals.yml`.
*
* Provider-specific fields are opaque to scan.mjs and validated by the
* provider itself. Examples in current providers: `api`, `careers_url`.
* Providers read these directly off the entry object — no schema enforcement
* at the framework level.
*
* @typedef {object} PortalEntry
* @property {string} name User-facing label; appears in logs and placeholders.
* @property {boolean} [enabled] Default: true.
* @property {string} [careers_url] Public listing URL; consumed by detect().
* @property {string} [api] JSON API URL; used directly by greenhouse/ashby providers.
* @property {string} [provider] Explicit provider id — bypasses detect().
* @property {('http')} [transport] Default: 'http'. Reserved for future transports.
* @property {number} [max_pages] Provider-specific pagination cap (avature, workday).
* @property {string} [offset_param] avature only: pins the pagination query key and disables the
* provider's jobOffset→offset self-heal. Rarely needed — an
* escape hatch for a tenant the auto-switch can't resolve.
*/
/**
* Returned by `detect()` when a provider claims an entry. `url` is
* informational (used in logs); routing only checks for a non-null return.
*
* @typedef {object} DetectHit
* @property {string} url
*/
/**
* Options forwarded to the underlying `fetch` call.
*
* @typedef {object} FetchOptions
* @property {number} [timeoutMs]
* @property {Object<string,string>} [headers]
* @property {string} [method]
* @property {(string|null)} [body]
* @property {('error'|'follow'|'manual')} [redirect]
*/
/**
* What scan.mjs hands to provider.fetch(). For Phase A only `transport: 'http'`
* is implemented; the shape reserves room for future transports without
* breaking the contract.
*
* @typedef {object} Context
* @property {('http')} transport
* @property {(url: string, opts?: FetchOptions) => Promise<string>} fetchText
* @property {(url: string, opts?: FetchOptions) => Promise<unknown>} fetchJson
* @property {number} [maxPages] Optional pagination hint. When set (verify-portals.mjs's
* health probe passes 1), a paginating provider SHOULD stop
* after this many pages — the probe only needs the first page
* to tell a live board from a broken one, and must not walk an
* entire careers site. Providers that ignore it stay correct:
* the probe caps their requests defensively via the context's
* own fetch functions.
* @property {(ms: number) => Promise<void>} [sleep] Optional cross-provider pacing hook used by
* paginating providers (avature, workday) to throttle between page
* requests. May be absent — providers fall back to a native
* `setTimeout`-based delay.
*/
/**
* The provider contract — the default export of every providers/*.mjs file
* (excluding _-prefixed shared helpers).
*
* @typedef {object} Provider
* @property {string} id Unique across all loaded providers.
* @property {((entry: PortalEntry) => (DetectHit | null))} [detect] Optional auto-detection.
* @property {(entry: PortalEntry, ctx: Context) => Promise<Job[]>} fetch Required.
*/
export {};
+107
View File
@@ -0,0 +1,107 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Amazon / AWS provider — hits the public amazon.jobs search JSON API.
// Auto-detects from a careers_url on `amazon.jobs`; the whole board is one
// global endpoint, so a tracked_companies entry narrows it with an `amazon:`
// config block whose keys are passed straight through as query params:
//
// - name: Amazon / AWS
// provider: amazon
// amazon:
// loc_query: Germany # free-text location filter
// base_query: machine learning # optional keyword filter
// category: software-development # optional facet (repeatable via array)
//
// The board is enormous (100k+ postings), so a location and/or keyword filter
// is effectively required — without one the MAX_PAGES cap just returns the most
// recent slice. result_limit is fixed at 100 (the API's hard per-page max;
// larger values return an empty `jobs`), and we page via `offset`.
const PAGE_SIZE = 100; // amazon.jobs caps result_limit at 100
const MAX_PAGES = 20; // safety cap — at most 2000 postings per entry
const ORIGIN = 'https://www.amazon.jobs';
/** @param {import('./_types.js').PortalEntry & {amazon?: Record<string, unknown>}} entry */
function buildQuery(entry, offset) {
const cfg = entry.amazon && typeof entry.amazon === 'object' ? entry.amazon : {};
const params = new URLSearchParams();
// Pass config keys through verbatim (base_query, loc_query, category, …).
// Array values are amazon.jobs facet filters and MUST use the `key[]=`
// bracket form to filter (e.g. normalized_country_code[]=DEU); a bare
// `key=DEU` is silently ignored and the board stays global.
for (const [k, v] of Object.entries(cfg)) {
if (v == null) continue;
if (Array.isArray(v)) {
const name = k.endsWith('[]') ? k : `${k}[]`;
for (const item of v) params.append(name, String(item));
} else {
params.append(k, String(v));
}
}
if (!params.has('base_query')) params.set('base_query', '');
if (!params.has('loc_query')) params.set('loc_query', '');
params.set('sort', cfg.sort ? String(cfg.sort) : 'recent');
params.set('result_limit', String(PAGE_SIZE));
params.set('offset', String(offset));
return `${ORIGIN}/en/search.json?${params.toString()}`;
}
// amazon.jobs posted_date reads "July 3, 2026" (note the padded day);
// Date.parse handles it once whitespace is collapsed. (updated_time is a
// relative string like "10 minutes" / "about 1 hour" — unparseable, skip it.)
function toEpochMs(job) {
const raw = job.posted_date;
if (!raw || typeof raw !== 'string') return undefined;
const parsed = Date.parse(raw.replace(/\s+/g, ' ').trim());
return Number.isNaN(parsed) ? undefined : parsed;
}
/** @type {Provider} */
export default {
id: 'amazon',
detect(entry) {
const url = entry.api || entry.careers_url || '';
if (typeof url !== 'string') return null;
// Match the host, not a path segment, to avoid spoofed URLs.
try {
const host = new URL(url).host.toLowerCase();
if (host === 'amazon.jobs' || host.endsWith('.amazon.jobs')) return { url };
} catch {
/* not an absolute URL */
}
return null;
},
async fetch(entry, ctx) {
const jobs = [];
const seen = new Set();
for (let page = 0; page < MAX_PAGES; page++) {
const url = buildQuery(entry, page * PAGE_SIZE);
const json = /** @type {any} */ (await ctx.fetchJson(url, { redirect: 'error' }));
const postings = Array.isArray(json?.jobs) ? json.jobs : [];
if (postings.length === 0) break;
let fresh = 0;
for (const j of postings) {
const path = j.job_path;
if (!path || typeof path !== 'string') continue;
const url2 = /^https?:\/\//i.test(path) ? path : ORIGIN + (path.startsWith('/') ? path : '/' + path);
if (seen.has(url2)) continue;
seen.add(url2);
fresh++;
jobs.push({
title: (j.title || '').trim(),
url: url2,
company: j.company_name || entry.name,
location: (j.normalized_location || j.location || '').trim(),
postedAt: toEpochMs(j),
});
}
if (fresh === 0) break; // API ignored offset / looped
if (postings.length < PAGE_SIZE) break; // last page
}
return jobs;
},
};
+127
View File
@@ -0,0 +1,127 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Arbeitnow provider — board-wide aggregator feed (EU/DACH-heavy, but
// international): https://www.arbeitnow.com/api/job-board-api
// Response shape: { data: [ { slug, company_name, title, description, remote,
// url, tags, job_types, location, created_at } ], links, meta }
//
// Jobs are ordered newest-first, 100 per page. The API echoes a rotating
// featured `?search=` term into links.next, so page URLs are built directly as
// `?page=N` (never by following links.next) to keep the FULL board in view —
// scan.mjs's title_filter then gates on the configured titles. Pages are fetched
// until one comes back short/empty or the page cap is reached (default 3,
// override with `max_pages` on the portal entry).
//
// Wire in via a `job_boards:` entry with `provider: arbeitnow`.
const FEED_BASE = 'https://www.arbeitnow.com/api/job-board-api';
const TRUSTED_HOST = 'www.arbeitnow.com';
const PER_PAGE = 100;
const DEFAULT_MAX_PAGES = 3;
const MAX_PAGES_CAP = 50;
/** @param {string} url */
function assertArbeitnowUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`arbeitnow: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`arbeitnow: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST) {
throw new Error(`arbeitnow: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
}
return url;
}
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
function resolveMaxPages(entry) {
const v = entry?.max_pages;
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
return DEFAULT_MAX_PAGES;
}
/**
* Normalize a single Arbeitnow job. Exported for unit tests.
*
* Field mapping → the normalized Job shape:
* - title: `title`, trimmed (items without one are dropped).
* - url: `url` — an absolute `https:` posting URL host-locked to
* www.arbeitnow.com (an off-host or non-https URL is untrusted and
* drops the item). It is the dedup key and is display-only (written
* to the pipeline/history, never server-fetched here).
* - company: `company_name`, falling back to the portal entry name, then
* "Arbeitnow".
* - location: `location`, with "Remote" appended when `remote` is true.
* - postedAt: `created_at` (epoch SECONDS) → epoch ms (omitted when absent).
*
* @param {any} j
* @param {string} [fallbackCompany]
* @returns {{ title: string, url: string, company: string, location: string, postedAt?: number } | null}
*/
export function normalizeArbeitnowJob(j, fallbackCompany) {
if (!j || typeof j !== 'object') return null;
const title = typeof j.title === 'string' ? j.title.trim() : '';
if (!title) return null;
// url must be an absolute https posting link on www.arbeitnow.com — Arbeitnow
// always serves postings there, so an off-host URL is untrusted and dropped.
let url = '';
const rawUrl = typeof j.url === 'string' ? j.url.trim() : '';
if (rawUrl) {
try {
const parsed = new URL(rawUrl);
if (parsed.protocol === 'https:' && parsed.hostname === TRUSTED_HOST) url = parsed.href;
} catch {
// malformed URL → leave url = '' → dropped below
}
}
if (!url) return null;
const company =
typeof j.company_name === 'string' && j.company_name.trim()
? j.company_name.trim()
: fallbackCompany || 'Arbeitnow';
const baseLocation = typeof j.location === 'string' ? j.location.trim() : '';
const location = [baseLocation, j.remote === true ? 'Remote' : ''].filter(Boolean).join(', ');
/** @type {{ title: string, url: string, company: string, location: string, postedAt?: number }} */
const job = { title, url, company, location };
if (Number.isFinite(j.created_at)) job.postedAt = j.created_at * 1000; // epoch seconds → ms
return job;
}
/** @type {Provider} */
export default {
id: 'arbeitnow',
async fetch(entry, ctx) {
assertArbeitnowUrl(FEED_BASE);
const maxPages = resolveMaxPages(entry);
const fallbackCompany = entry?.name;
const out = [];
for (let page = 1; page <= maxPages; page++) {
// Build the page URL directly (do NOT follow links.next — it carries a
// featured `?search=` term that would narrow the board).
const url = `${FEED_BASE}?page=${page}`;
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(url, { redirect: 'error' });
if (!json || !Array.isArray(json.data)) {
throw new Error(
`arbeitnow: unexpected API response on page ${page} — expected { data: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`,
);
}
for (const j of json.data) {
const normalized = normalizeArbeitnowJob(j, fallbackCompany);
if (normalized) out.push(normalized);
}
if (json.data.length < PER_PAGE) break; // short page → last page reached
}
return out;
},
};
+200
View File
@@ -0,0 +1,200 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Arbeitsagentur (Bundesagentur für Arbeit) provider — hits the public Jobsuche
// REST API (the same endpoint arbeitsagentur.de uses), so it lives in-process
// alongside the other JSON-API providers (greenhouse/ashby shape). One or more
// keywords are queried; scan.mjs applies title_filter + location_filter + dedup
// afterwards, so this provider over-fetches (recall-first).
//
// Configure via a `job_boards` (or `tracked_companies`) entry with
// `provider: arbeitsagentur` and an `arbeitsagentur:` block:
//
// - name: Arbeitsagentur — ML/KI Deutschland
// provider: arbeitsagentur
// arbeitsagentur:
// keywords: ["Machine Learning Engineer", "Data Scientist"] # required
// wo: Berlin # optional anchor city; omit for nationwide
// umkreis: 50 # km radius around `wo` (default 50)
// days: 30 # recency window in days (default 30)
// size: 100 # results per keyword (1–100, default 100)
// remoteNationwide: true # also run a nationwide pass keeping remote-eligible hits
// remoteMatch: filter # how that pass detects remote (default 'title'):
// # 'filter' — server-side `homeoffice=nv_true` query + pagination; every hit is
// # remote-eligible, cheap (no per-job calls). Recommended.
// # 'title' — regex on the job title only (cheap; misses body-level remote)
// # 'off' — skip the remote pass entirely
// remoteMaxPages: 10 # 'filter' mode: max pages to paginate (size each); default 1
// enabled: true
const API_URL = 'https://rest.arbeitsagentur.de/jobboerse/jobsuche-service/pc/v4/jobs';
const API_KEY = 'jobboerse-jobsuche'; // public client key the arbeitsagentur.de UI uses
const DETAIL_BASE = 'https://www.arbeitsagentur.de/jobsuche/jobdetail/';
const REMOTE_RE = /(remote|homeoffice|home[-\s]?office|ortsunabh|deutschlandweit|bundesweit|100\s*%|full[-\s]?remote|fully remote)/i;
// Clamp a runtime integer into [min, max], falling back to `def` for NaN, so a
// stray portals.yml value can't produce empty (size=0) or pathological queries.
function intInRange(val, def, min, max) {
const n = Number(val);
if (!Number.isFinite(n)) return def;
return Math.min(max, Math.max(min, Math.trunc(n)));
}
/**
* Reads and sanitizes the entry's `arbeitsagentur:` config block.
* @param {{ arbeitsagentur?: any }} entry
* @returns {{ keywords: string[], wo: string, umkreis: number, days: number, size: number, remoteNationwide: boolean, remoteMatch: 'title'|'filter'|'off', remoteMaxPages: number }}
*/
export function parseArbeitsagenturConfig(entry) {
const cfg = (entry && entry.arbeitsagentur) || {};
const keywords = Array.isArray(cfg.keywords)
? cfg.keywords.filter(k => typeof k === 'string' && k.trim()).map(k => k.trim())
: [];
return {
keywords,
wo: typeof cfg.wo === 'string' ? cfg.wo.trim() : '',
umkreis: intInRange(cfg.umkreis, 50, 0, 1000), // km; only used when `wo` is set
days: intInRange(cfg.days, 30, 1, 1000), // recency window
size: intInRange(cfg.size, 100, 1, 100), // results per keyword (API max 100)
remoteNationwide: cfg.remoteNationwide === true,
// Remote-detection mode is config-driven (not hardcoded).
remoteMatch: ['title', 'filter', 'off'].includes(cfg.remoteMatch) ? cfg.remoteMatch : 'title',
remoteMaxPages: intInRange(cfg.remoteMaxPages, 1, 1, 20),
};
}
/**
* Assembles a human-readable location from the API's `arbeitsort` object. Most
* postings are in Germany; only a non-DE country is appended so the downstream
* location_filter can act on it.
* @param {any} arbeitsort
*/
export function buildLocation(arbeitsort) {
if (!arbeitsort || typeof arbeitsort !== 'object') return '';
const loc = [arbeitsort.ort, arbeitsort.region].filter(Boolean).join(', ');
const land = arbeitsort.land;
if (land && !/deutschland|germany/i.test(land)) return loc ? `${loc}, ${land}` : land;
return loc;
}
/**
* Normalizes one raw Arbeitsagentur posting into a Job plus its `refnr` (kept
* for dedup, stripped before the provider returns). Returns null when the
* posting lacks a usable refnr or title.
* @param {any} job
* @returns {({title: string, url: string, company: string, location: string, refnr: string}) | null}
*/
export function normalizeJob(job) {
const refnr = job && job.refnr;
const title = String((job && job.titel) || '').trim();
if (!refnr || !title) return null;
return {
title,
url: DETAIL_BASE + encodeURIComponent(String(refnr)),
company: String((job && job.arbeitgeber) || '').trim(),
location: buildLocation(job && job.arbeitsort),
refnr: String(refnr),
};
}
/** @type {Provider} */
export default {
id: 'arbeitsagentur',
/**
* Fetches and normalizes postings from the Arbeitsagentur Jobsuche API.
* @param {{ name?: string, arbeitsagentur?: any }} entry
* @param {{ fetchJson: (url: string, opts?: object) => Promise<any> }} ctx
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>}
*/
async fetch(entry, ctx) {
const { keywords, wo, umkreis, days, size, remoteNationwide, remoteMatch, remoteMaxPages } = parseArbeitsagenturConfig(entry);
if (!keywords.length) {
throw new Error(`arbeitsagentur: entry "${entry.name || '(unnamed)'}" has no arbeitsagentur.keywords[]`);
}
/** @param {string} was @param {Record<string,string>} [extra] */
const fetchKeyword = async (was, extra = {}) => {
const params = new URLSearchParams({
was,
size: String(size),
page: '1',
angebotsart: '1', // 1 = ARBEIT (employment; excludes Ausbildung/Selbständigkeit)
veroeffentlichtseit: String(days),
...extra,
});
// redirect:'error' prevents SSRF via server-side redirects.
const json = await ctx.fetchJson(`${API_URL}?${params.toString()}`, {
headers: { 'X-API-Key': API_KEY, accept: 'application/json' },
redirect: 'error',
timeoutMs: 12_000,
});
return Array.isArray(json && json.stellenangebote) ? json.stellenangebote : [];
};
const byRef = new Map();
const errors = [];
let succeeded = 0; // keywords whose primary pass completed (i.e. the source answered)
for (const kw of keywords) {
let primary;
try {
// Pass A: commutable radius around `wo`, or a single nationwide pass.
primary = wo
? await fetchKeyword(kw, { wo, umkreis: String(umkreis) })
: await fetchKeyword(kw);
succeeded++;
} catch (err) {
// Recall-first: tolerate a single failed keyword and keep going.
errors.push(`"${kw}": ${(err && err.message) || err}`);
continue;
}
// Pass B (optional): a nationwide pass for remote roles hosted at a far HQ
// (which the radius pass misses). Detection is config-driven via `remoteMatch`:
// 'filter' — server-side `homeoffice=nv_true` query + pagination (every hit is remote)
// 'title' — keep only nationwide hits whose title matches the remote regex
// Its failure must NOT discard the primary results already fetched above.
let wide = [];
if (wo && remoteNationwide && remoteMatch !== 'off') {
try {
if (remoteMatch === 'filter') {
// Server-side home-office filter: every hit is remote-eligible, so just
// paginate and keep them all — no per-job calls, no title regex.
for (let page = 1; page <= remoteMaxPages; page++) {
const res = await fetchKeyword(kw, { homeoffice: 'nv_true', page: String(page) });
wide.push(...res);
if (res.length < size) break; // short page → done
}
} else { // 'title'
const nationwide = await fetchKeyword(kw);
wide = nationwide.filter(j => REMOTE_RE.test(String((j && j.titel) || '')));
}
} catch (err) {
errors.push(`"${kw}" (remote pass): ${(err && err.message) || err}`);
}
}
// Pass A (commutable) keeps its city as-is. Pass B roles are remote, so we
// append a `Deutschlandweit (Homeoffice)` marker — remote ignores distance,
// and this lets scan.mjs's commute-based location_filter pass them via its
// always_allow rescue instead of dropping them on the far office city.
for (const raw of primary) {
const job = normalizeJob(raw);
if (job && !byRef.has(job.refnr)) byRef.set(job.refnr, job);
}
for (const raw of wide) {
const job = normalizeJob(raw);
if (!job) continue;
job.location = job.location ? `${job.location} · Deutschlandweit (Homeoffice)` : 'Deutschlandweit (Homeoffice)';
if (!byRef.has(job.refnr)) byRef.set(job.refnr, job);
}
}
// Total outage = every primary request failed. A keyword that answered with
// zero results is not an outage, so key off the success count, not the
// deduped result size — otherwise a legitimately-empty search throws.
if (succeeded === 0 && errors.length) {
throw new Error(`arbeitsagentur: all ${keywords.length} keyword request(s) failed — ${errors[0]}`);
}
return [...byRef.values()].map(({ refnr, ...job }) => job);
},
};
+157
View File
@@ -0,0 +1,157 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Ashby provider — hits the public posting-api endpoint.
// Auto-detects from careers_url pattern `https://jobs.ashbyhq.com/<slug>`.
//
// Ashby's public posting-api carries a ~10s+ server-side latency floor
// (response time is independent of board size) and rate-limits repeated
// unauthenticated hits. The global default timeout (10s, providers/_http.mjs)
// sits right on that floor, so requests race the timeout and abort. We give
// Ashby a longer timeout plus a backoff+jitter retry (the backoff spaces
// requests out to dodge rate-limiting).
// See .planning/codebase/ashby-scan-abort-diagnosis.md.
const ASHBY_TIMEOUT_MS = 30_000;
const ASHBY_RETRIES = 2;
// Annualization multipliers for different compensation intervals
const INTERVAL_MULTIPLIERS = {
'1 HOUR': 2080,
'1 DAY': 260,
'1 WEEK': 52,
'2 WEEK': 26,
'0.5 MONTH': 24,
'1 MONTH': 12,
'2 MONTH': 6,
'3 MONTH': 4,
'6 MONTH': 2,
'1 YEAR': 1,
};
/**
* Parse compensation data from Ashby job object.
* Returns structured salary object with min, max, and currency,
* or null if no valid compensation data exists.
* @param {any} job - Ashby job object
* @returns {{min: number, max: number, currency: string}|null}
*/
export function parseCompensation(job) {
const comp = job?.compensation;
if (!comp) return null;
const interval = /** @type {keyof typeof INTERVAL_MULTIPLIERS} */ (comp.interval || '1 YEAR');
const multiplier = INTERVAL_MULTIPLIERS[interval];
if (!multiplier) return null;
// Coerce and validate numeric fields — malformed API payloads must not propagate
/** @param {any} v */
const normalizeNum = (v) => {
if (v == null) return null;
if (typeof v === 'string' && v.trim() === '') return null;
const n = Number(v);
return Number.isFinite(n) && n >= 0 ? n : null;
};
const minValue = normalizeNum(comp.minValue);
const maxValue = normalizeNum(comp.maxValue);
const currency = typeof comp.currency === 'string' ? comp.currency.trim() : '';
// If neither min nor max is provided, no valid compensation
if (minValue == null && maxValue == null) return null;
// Annualize the values
const min = minValue != null ? minValue * multiplier : null;
const max = maxValue != null ? maxValue * multiplier : null;
// Must have at least one valid annual value
if (min == null && max == null) return null;
// Ensure correct ordering (min <= max)
const resolvedMin = /** @type {number} */ (min ?? max);
const resolvedMax = /** @type {number} */ (max ?? min);
return {
min: Math.min(resolvedMin, resolvedMax),
max: Math.max(resolvedMin, resolvedMax),
currency: currency.toUpperCase(),
};
}
/** @param {import('./_types.js').PortalEntry} entry */
function resolveApiUrl(entry) {
const url = entry.careers_url || '';
const match = url.match(/jobs\.ashbyhq\.com\/([^/?#]+)/);
if (!match) return null;
return `https://api.ashbyhq.com/posting-api/job-board/${match[1]}?includeCompensation=true`;
}
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
// Build the full location string from primary + secondary locations.
// Ashby's posting-api puts extra hiring regions in `secondaryLocations[]`
// (each with a region label + a postalAddress). Using only `j.location` drops
// them, so an EU-eligible role whose PRIMARY label is e.g. "Canada" reads as
// Canada-only and gets wrongly removed by scan.mjs's location_filter. We fold
// in each secondary's region, locality, and country so the filter can match
// (e.g. "Europe", "Berlin", "Germany"). Deduped, joined with " · ".
/** @param {any} j */
function formatLocation(j) {
const parts = [];
if (typeof j.location === 'string' && j.location.trim()) parts.push(j.location.trim());
if (Array.isArray(j.secondaryLocations)) {
for (const s of j.secondaryLocations) {
if (!s || typeof s !== 'object') continue;
if (typeof s.location === 'string' && s.location.trim()) parts.push(s.location.trim());
const pa = s.address && s.address.postalAddress;
if (pa) {
for (const k of ['addressLocality', 'addressCountry']) {
if (typeof pa[k] === 'string' && pa[k].trim()) parts.push(pa[k].trim());
}
}
}
}
return [...new Set(parts)].join(' · ');
}
/** @type {Provider} */
export default {
id: 'ashby',
detect(entry) {
const apiUrl = resolveApiUrl(entry);
return apiUrl ? { url: apiUrl } : null;
},
async fetch(entry, ctx) {
const apiUrl = resolveApiUrl(entry);
if (!apiUrl) throw new Error(`ashby: cannot derive API URL for ${entry.name}`);
let lastErr;
for (let attempt = 0; attempt <= ASHBY_RETRIES; attempt++) {
if (attempt > 0) {
// exponential backoff + jitter — spaces out retries to dodge Ashby rate-limiting
const backoff = 1000 * 2 ** (attempt - 1) + Math.floor(Math.random() * 500);
await sleep(backoff);
}
try {
const json = /** @type {any} */ (await ctx.fetchJson(apiUrl, { timeoutMs: ASHBY_TIMEOUT_MS, redirect: 'error' }));
const jobs = Array.isArray(json?.jobs) ? json.jobs : [];
return jobs.map(/** @param {any} j */ (j) => ({
title: j.title || '',
url: j.jobUrl || '',
company: entry.name,
location: formatLocation(j),
salary: parseCompensation(j),
postedAt: toEpochMs(j.publishedAt),
}));
} catch (e) {
lastErr = e;
}
}
throw lastErr;
},
};
+224
View File
@@ -0,0 +1,224 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Avature provider — parses the public Avature career-site job list.
// Auto-detects from a careers_url on `*.avature.net`; a branded custom domain
// that proxies Avature needs an explicit `provider: avature` + `api:` pointing
// at the Avature origin (or its /careers/SearchJobs URL).
//
// GET {origin}/careers/SearchJobs?jobOffset=N
//
// returns a server-rendered page of <article class="article--result"> blocks.
// Avature hard-caps the page at 6 results (a jobRecordsPerPage override is
// ignored), and paginates via jobOffset in steps of 6 — so this is request-
// heavy for large boards; max_pages (default 50 → ~300 postings) bounds it and
// can be raised per entry.
//
// Pagination parameter name is `jobOffset` on classic tenants, but some branded
// tenants ignore it and page via a bare `offset` instead (e.g. jobs.siemens.com:
// `?jobOffset=6` returns page 1 unchanged, `?offset=6` advances). This is
// self-healing: we start with `jobOffset`, and if the first paginated page comes
// back identical to page 0 (every id already seen → the param is inert), we
// retry that page once with `offset` and adopt whichever key actually advances.
// So a new branded tenant needs no configuration. An entry can still pin the key
// explicitly via `offset_param` (disables the auto-switch) as an escape hatch.
//
// Location isn't rendered in every tenant's list (the subtitle is Job ID /
// hire-type / posted-date); we extract it when a marker is present and leave it
// empty otherwise. postedAt comes from the "Posted DD-Mon-YYYY" subtitle.
const PAGE_SIZE = 6; // Avature serves exactly 6 results per page
const DEFAULT_MAX_PAGES = 50; // ~300 postings; override via entry.max_pages
const HARD_MAX_PAGES = 200;
// Pause between successive page requests. Avature's 6-results-per-page cap makes
// large boards request-heavy (a 999+ board is ~170 pages); firing those with no
// gap risks the tenant's WAF rate-limiting the burst. Mirrors workday's
// INTER_PAGE_DELAY_MS — only boards that paginate past page 0 pay it.
const INTER_PAGE_DELAY_MS = 150;
// The bare key we self-heal to when the primary (`jobOffset`) proves inert.
const FALLBACK_OFFSET_PARAM = 'offset';
const MONTHS = { jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5, jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11 };
/** @param {import('./_types.js').PortalEntry} entry */
function resolveConfig(entry) {
const raw = entry.api || entry.careers_url || '';
let u;
try {
u = new URL(raw);
} catch {
return null;
}
if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
// Honour an explicit SearchJobs path (branded tenants may prefix a locale,
// e.g. /en_US/searchjobs/SearchJobs); otherwise default to the classic path.
const searchPath = /\/SearchJobs\b/i.test(u.pathname) ? u.pathname.replace(/\/+$/, '') : '/careers/SearchJobs';
return { searchUrl: `${u.origin}${searchPath}`, origin: u.origin };
}
const NAMED = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', '#8226': '•' };
/** @param {string} s */
function decodeEntities(s) {
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z0-9]+);/g, (m, body) => {
if (body[0] === '#') {
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
}
return NAMED[body.toLowerCase()] ?? m;
});
}
/** @param {string} s */
function clean(s) {
return decodeEntities(s.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
}
// "Posted 02-May-2026" → epoch ms (UTC midnight). Undefined when absent/unparseable.
/** @param {string} block */
function parsePosted(block) {
const m = block.match(/Posted\s+(\d{1,2})-([A-Za-z]{3})-(\d{4})/);
if (!m) return undefined;
const mon = MONTHS[m[2].toLowerCase()];
if (mon === undefined) return undefined;
const ms = Date.UTC(Number(m[3]), mon, Number(m[1]));
return Number.isNaN(ms) ? undefined : ms;
}
// Best-effort location: some tenants tag it with a list-item-location span or a
// map-marker glyph; most (e.g. Synopsys) render none, so this returns ''.
/** @param {string} block */
function parseLocation(block) {
const m =
block.match(/list-item-location[^>]*>([\s\S]*?)<\/span>/i) ||
block.match(/class="[^"]*\blocation\b[^"]*"[^>]*>([\s\S]*?)<\/(?:span|div|li)>/i) ||
block.match(/glyphicon-map-marker[\s\S]{0,80}?>([^<]{2,60})</i);
return m ? clean(m[1]) : '';
}
/** @param {string} htmlText @param {string} origin */
export function parseArticles(htmlText, origin) {
const out = [];
// Tenants vary the result class: Synopsys uses `article--result`, Siemens
// appends a position index (`article--result 1`). Accept any suffix.
const re = /<article class="article article--result[^"]*"[\s\S]*?<\/article>/g;
let a;
while ((a = re.exec(htmlText)) !== null) {
const block = a[0];
// JobDetail path may or may not sit under /careers/ (branded tenants vary),
// so anchor on JobDetail/ itself rather than a fixed prefix. Prefer the
// `class="link"` title anchor (most tenants); fall back to any JobDetail
// anchor for tenants (e.g. Rohde & Schwarz) whose title link carries no
// class. Share/mailto buttons url-encode the path (%2FJobDetail%2F) so they
// never match the literal `/JobDetail/` and can't be mistaken for the title.
const urlM =
block.match(/<a[^>]*class="link"[^>]*href="([^"]*\/JobDetail\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/) ||
block.match(/<a[^>]*href="([^"]*\/JobDetail\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/);
if (!urlM) continue;
const title = clean(urlM[2]);
if (!title) continue;
let url = decodeEntities(urlM[1]);
if (!/^https?:\/\//i.test(url)) url = origin + (url.startsWith('/') ? url : '/' + url);
const idM = url.match(/\/JobDetail\/[^/]*\/(\d+)/);
out.push({
id: idM ? idM[1] : url,
title,
url,
location: parseLocation(block),
postedAt: parsePosted(block),
});
}
return out;
}
/** @type {Provider} */
export default {
id: 'avature',
detect(entry) {
const url = entry.api || entry.careers_url || '';
if (typeof url !== 'string') return null;
try {
const host = new URL(url).host.toLowerCase();
if (host === 'avature.net' || host.endsWith('.avature.net')) return { url };
} catch {
/* not absolute */
}
return null;
},
async fetch(entry, ctx) {
const cfg = resolveConfig(entry);
if (!cfg) throw new Error(`avature: cannot resolve origin for ${entry.name}`);
const maxPages = Math.min(
HARD_MAX_PAGES,
Number.isFinite(entry.max_pages) && entry.max_pages > 0 ? Number(entry.max_pages) : DEFAULT_MAX_PAGES,
);
// Pagination key. An explicit `offset_param` pins it (and disables the
// auto-switch below); otherwise start with `jobOffset` and self-heal. A
// non-string/empty override falls back to the default so a malformed entry
// can't produce `?=N`.
const pinned = typeof entry.offset_param === 'string' && entry.offset_param.trim();
let offsetParam = pinned ? entry.offset_param.trim() : 'jobOffset';
let canHeal = !pinned; // once the key is pinned, never auto-switch
const jobs = [];
const seen = new Set();
const sleep = (ms) => (typeof ctx?.sleep === 'function' ? ctx.sleep(ms) : new Promise((r) => setTimeout(r, ms)));
const getPage = async (param, page) => {
const htmlText = await ctx.fetchText(`${cfg.searchUrl}?${param}=${page * PAGE_SIZE}`, {
redirect: 'error',
headers: { accept: 'text/html' },
});
return parseArticles(htmlText, cfg.origin);
};
// Absorb a page's articles, returning how many were not already seen.
const absorb = (articles) => {
let fresh = 0;
for (const art of articles) {
if (seen.has(art.id)) continue;
seen.add(art.id);
fresh++;
jobs.push({
title: art.title,
url: art.url,
company: entry.name,
location: art.location,
postedAt: art.postedAt,
});
}
return fresh;
};
for (let page = 0; page < maxPages; page++) {
if (page > 0) await sleep(INTER_PAGE_DELAY_MS);
let articles = await getPage(offsetParam, page);
let fresh = absorb(articles);
// Self-heal: the first paginated page didn't advance — either it repeated
// page 0 (all-dup) or came back empty because the primary key is inert
// (some tenants echo page 0 for an unknown param, others return nothing).
// Retry this page once with the fallback key; if it advances, adopt it for
// the remainder. Only on page 1 — inert pagination manifests immediately,
// so a later non-advancing page is a genuine end-of-board, not a mismatch.
// NOTE: fresh === 0 must be evaluated before the empty-page break below,
// or an inert key that returns an empty page 1 would exit without healing.
if (fresh === 0 && canHeal && page === 1) {
canHeal = false;
await sleep(INTER_PAGE_DELAY_MS);
const altArticles = await getPage(FALLBACK_OFFSET_PARAM, page);
const altFresh = absorb(altArticles);
if (altFresh > 0) {
offsetParam = FALLBACK_OFFSET_PARAM;
articles = altArticles;
fresh = altFresh;
}
}
if (fresh === 0) break; // empty page / looped / offset ignored / last page
if (articles.length < PAGE_SIZE) break; // last page
}
return jobs;
},
};
+111
View File
@@ -0,0 +1,111 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// BambooHR provider — hits the public per-tenant careers list API.
// Auto-detects from careers_url pattern `https://<tenant>.bamboohr.com[/...]`.
// Per-tenant subdomains are the variable part, so SSRF defence uses a regex
// match on `<safe-tenant>.bamboohr.com` rather than a static allowlist
// (same approach as the recruitee provider).
//
// The list endpoint (`/careers/list`) returns lightweight metadata — enough for
// the Job contract (title, url, location) at zero token cost. The full JD lives
// behind a second `/careers/<id>/detail` request, which the scanner deliberately
// skips to stay zero-token (so `description`/`postedAt` are omitted).
const BAMBOOHR_HOST_RE = /^[a-z0-9][a-z0-9-]*\.bamboohr\.com$/;
/** @param {string} url */
function assertBambooHRUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`bamboohr: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`bamboohr: URL must use HTTPS: ${url}`);
if (!BAMBOOHR_HOST_RE.test(parsed.hostname)) {
throw new Error(`bamboohr: untrusted hostname "${parsed.hostname}" — must match <tenant>.bamboohr.com`);
}
return url;
}
/**
* Resolve the tenant origin (`https://<tenant>.bamboohr.com`) from an entry.
* Honours an explicit `api:` URL, else parses `careers_url`.
* @param {import('./_types.js').PortalEntry} entry
* @returns {string | null}
*/
function resolveOrigin(entry) {
const rawApi = typeof entry.api === 'string' ? entry.api : '';
const rawCareers = typeof entry.careers_url === 'string' ? entry.careers_url : '';
const raw = (rawApi || rawCareers).trim();
if (!raw) return null;
let parsed;
try {
parsed = new URL(raw);
} catch {
return null;
}
if (parsed.protocol !== 'https:') return null;
if (!BAMBOOHR_HOST_RE.test(parsed.hostname)) return null;
return `https://${parsed.hostname}`;
}
/** @type {Provider} */
export default {
id: 'bamboohr',
detect(entry) {
const origin = resolveOrigin(entry);
return origin ? { url: `${origin}/careers/list` } : null;
},
async fetch(entry, ctx) {
const origin = resolveOrigin(entry);
if (!origin) throw new Error(`bamboohr: cannot derive API URL for ${entry.name}`);
const apiUrl = `${origin}/careers/list`;
assertBambooHRUrl(apiUrl);
// redirect:'error' + the host check above keep the final hostname pinned to
// the tenant — a server-side redirect can't bounce us off-domain (SSRF).
const json = /** @type {any} */ (await ctx.fetchJson(apiUrl, { redirect: 'error' }));
return parseBambooHRResponse(json, entry.name, origin);
},
};
/**
* Parse a BambooHR `/careers/list` response. Exported for unit tests.
*
* BambooHR returns:
* { meta: {...}, result: [{ id, jobOpeningName,
* location: { city?, state? }, isRemote?, employmentStatusLabel? }] }
*
* - url: built as `<origin>/careers/<id>` — matches the public
* `jobOpeningShareUrl`. Rows without a non-empty `id` are dropped (no stable
* URL, and url is the scanner's dedup key — a blank id would emit
* `/careers/` and collapse distinct postings together).
* - location: join `city` + `state`; append "Remote" when `isRemote` is truthy.
* BambooHR's `isRemote` is `1`/`true` when set and `null` otherwise.
*
* @param {any} json
* @param {string} companyName
* @param {string} origin e.g. "https://acme.bamboohr.com"
* @returns {Array<{title: string, url: string, company: string, location: string}>}
*/
export function parseBambooHRResponse(json, companyName, origin) {
const rows = json?.result;
if (!Array.isArray(rows)) return [];
return rows
.filter(j => j && j.jobOpeningName && String(j.id ?? '').trim().length > 0)
.map(j => {
const loc = j.location || {};
const remote = j.isRemote ? 'Remote' : '';
const location = [loc.city, loc.state, remote].filter(Boolean).join(', ');
const id = String(j.id).trim();
return {
title: String(j.jobOpeningName),
url: `${origin}/careers/${encodeURIComponent(id)}`,
company: companyName,
location,
};
});
}
+180
View File
@@ -0,0 +1,180 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// beesite (milch & zucker GJB) provider — the search backend behind branded
// portals like jobs.mercedes-benz.com. The SPA calls a public, no-auth JSON
// endpoint on the tenant's *.app.beesite.de host:
//
// GET {origin}/search?data={URL-encoded JSON}
// data = {"LanguageCode":"EN",
// "SearchParameters":{"FirstItem":1,"CountItem":100,
// "Sort":[{"Criterion":"PublicationStartDate","Direction":"DESC"}],
// "MatchedObjectDescriptor":[…field list…]},
// "SearchCriteria":[…optional facet filters…]}
// → {"SearchResult":{"SearchResultCount":N,"SearchResultCountAll":TOTAL,
// "SearchResultItems":[{"MatchedObjectId":"200325",
// "MatchedObjectDescriptor":{"PositionID":"mer0003yhd",
// "PositionTitle":…,"PositionURI":"https://jobs.mercedes-benz.com/…",
// "PositionLocation":[{"CityName":"Bremen"}],
// "PublicationStartDate":"2026-07-04"}}]}}
//
// PositionURI is the BRANDED job page (jobs.mercedes-benz.com/…), so listings
// link straight to the public posting. FirstItem is 1-based; requesting past
// the end returns an empty item list. We sort newest-first so a bounded walk
// (max_pages / MAX_JOBS) keeps the most recent postings.
//
// Facet criteria use tenant-internal numeric term codes (Mercedes: country
// 329 = Germany, discoverable via MatchedObjectDescriptor "Facet:…" queries).
// A portals.yml entry can pin them via:
// beesite:
// searchCriteria:
// - { CriterionName: PositionLocation.Country, CriterionValue: [329] }
const PAGE_SIZE = 100; // verified: the endpoint happily serves 100+/page
const MAX_PAGES = 40; // safety cap on request count (40*100 = 4000 postings)
const MAX_JOBS = 1000; // cap total postings pulled (newest-first sort)
const PAGE_DELAY_MS = 150; // polite pacing between page requests
const DESCRIPTOR = [
'PositionID',
'PositionTitle',
'PositionURI',
'PositionLocation.CityName',
'PublicationStartDate',
];
/** @param {import('./_types.js').PortalEntry} entry */
export function resolveConfig(entry) {
const raw = entry.api || entry.careers_url || '';
let u;
try {
u = new URL(raw);
} catch {
return null;
}
if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
const host = u.host.toLowerCase();
if (host !== 'beesite.de' && !host.endsWith('.beesite.de')) return null;
const cfgBlock = entry.beesite && typeof entry.beesite === 'object' ? entry.beesite : {};
return {
searchApi: `${u.origin}/search`,
languageCode: typeof cfgBlock.languageCode === 'string' ? cfgBlock.languageCode : 'EN',
searchCriteria: Array.isArray(cfgBlock.searchCriteria) ? cfgBlock.searchCriteria : [],
};
}
/** Build the ?data= payload for one page. @param {any} cfg @param {number} firstItem */
export function buildSearchUrl(cfg, firstItem) {
const data = {
LanguageCode: cfg.languageCode,
SearchParameters: {
FirstItem: firstItem,
CountItem: PAGE_SIZE,
Sort: [{ Criterion: 'PublicationStartDate', Direction: 'DESC' }],
MatchedObjectDescriptor: DESCRIPTOR,
},
SearchCriteria: cfg.searchCriteria,
};
return `${cfg.searchApi}?data=${encodeURIComponent(JSON.stringify(data))}`;
}
// "2026-07-04" → epoch ms (UTC for determinism).
/** @param {unknown} raw @returns {number | undefined} */
export function parseBeesiteDate(raw) {
const m = typeof raw === 'string' ? raw.trim().match(/^(\d{4})-(\d{2})-(\d{2})$/) : null;
if (!m) return undefined;
const month = Number(m[2]);
const day = Number(m[3]);
if (month < 1 || month > 12 || day < 1 || day > 31) return undefined;
const ms = Date.UTC(Number(m[1]), month - 1, day);
return Number.isFinite(ms) ? ms : undefined;
}
/**
* Map one search response to raw {id, title, url, location, postedAt} records.
* @param {any} json
* @returns {{total: number|null, rows: Array<{id:string,title:string,url:string,location:string,postedAt?:number}>}}
*/
export function parseSearchResult(json) {
const sr = json?.SearchResult;
const total = typeof sr?.SearchResultCountAll === 'number' ? sr.SearchResultCountAll : null;
const items = Array.isArray(sr?.SearchResultItems) ? sr.SearchResultItems : [];
const rows = [];
for (const item of items) {
const d = item?.MatchedObjectDescriptor;
if (!d) continue;
const id = item.MatchedObjectId != null ? String(item.MatchedObjectId) : String(d.PositionID || '');
const title = String(d.PositionTitle || '').replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
const url = String(d.PositionURI || '').trim();
if (!id || !title || !/^https?:\/\//i.test(url)) continue;
const locs = Array.isArray(d.PositionLocation) ? d.PositionLocation : [];
const cities = [];
for (const l of locs) {
const c = String(l?.CityName || '').trim();
if (c && !cities.includes(c)) cities.push(c);
}
rows.push({
id,
title,
url,
location: cities.join(' / '),
postedAt: parseBeesiteDate(d.PublicationStartDate),
});
}
return { total, rows };
}
/** Resolve the page cap: positive integer `max_pages`, else default. */
function resolveMaxPages(entry) {
const v = entry?.max_pages;
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES);
return MAX_PAGES;
}
/** @type {Provider} */
export default {
id: 'beesite',
detect(entry) {
const url = entry.api || entry.careers_url || '';
if (typeof url !== 'string') return null;
return resolveConfig({ api: url }) ? { url } : null;
},
async fetch(entry, ctx) {
const cfg = resolveConfig(entry);
if (!cfg) throw new Error(`beesite: cannot resolve search host for ${entry.name}`);
const wait = (ms) => (ctx.sleep ? ctx.sleep(ms) : new Promise((r) => setTimeout(r, ms)));
const maxPages = resolveMaxPages(entry);
const jobs = [];
const seen = new Set();
let total = null;
for (let page = 0; page < maxPages; page++) {
if (page > 0) await wait(PAGE_DELAY_MS);
const json = await ctx.fetchJson(buildSearchUrl(cfg, page * PAGE_SIZE + 1), {
redirect: 'error',
headers: { accept: 'application/json' },
});
const { total: pageTotal, rows } = parseSearchResult(json);
if (total === null) total = pageTotal;
if (rows.length === 0) break; // past the last page
let fresh = 0;
for (const row of rows) {
if (seen.has(row.id)) continue;
seen.add(row.id);
fresh++;
const job = { title: row.title, url: row.url, company: entry.name, location: row.location };
if (typeof row.postedAt === 'number') job.postedAt = row.postedAt;
jobs.push(job);
if (jobs.length >= MAX_JOBS) break;
}
if (fresh === 0) break; // server ignored FirstItem (or we've looped)
if (jobs.length >= MAX_JOBS) break;
if (total !== null && (page + 1) * PAGE_SIZE >= total) break;
}
return jobs;
},
};
+136
View File
@@ -0,0 +1,136 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Breezy HR provider — hits the public per-tenant board feed.
// Auto-detects from careers_url pattern `https://<tenant>.breezy.hr[/...]`.
// Per-tenant subdomains are the variable part, so SSRF defence uses a regex
// match on `<safe-tenant>.breezy.hr` rather than a static allowlist (same
// approach as the recruitee / bamboohr providers).
//
// Breezy boards expose every published position as a public JSON array at
// `<tenant>.breezy.hr/json` — title, absolute url, location, and a published
// date, all in the list payload at zero token cost (no per-job request, so the
// scanner stays zero-token). Breezy's authenticated REST API (api.breezy.hr) is
// intentionally NOT used; only the public board feed.
const BREEZY_HOST_RE = /^[a-z0-9][a-z0-9-]*\.breezy\.hr$/;
/** @param {string} url */
function assertBreezyUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`breezy: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`breezy: URL must use HTTPS: ${url}`);
if (!BREEZY_HOST_RE.test(parsed.hostname)) {
throw new Error(`breezy: untrusted hostname "${parsed.hostname}" — must match <tenant>.breezy.hr`);
}
return url;
}
/**
* Resolve the tenant origin (`https://<tenant>.breezy.hr`) from an entry.
* Honours an explicit `api:` URL, else parses `careers_url`.
* @param {import('./_types.js').PortalEntry} entry
* @returns {string | null}
*/
function resolveOrigin(entry) {
const rawApi = typeof entry.api === 'string' ? entry.api : '';
const rawCareers = typeof entry.careers_url === 'string' ? entry.careers_url : '';
const raw = (rawApi || rawCareers).trim();
if (!raw) return null;
let parsed;
try {
parsed = new URL(raw);
} catch {
return null;
}
if (parsed.protocol !== 'https:') return null;
if (!BREEZY_HOST_RE.test(parsed.hostname)) return null;
return `https://${parsed.hostname}`;
}
/** @type {Provider} */
export default {
id: 'breezy',
detect(entry) {
const origin = resolveOrigin(entry);
return origin ? { url: `${origin}/json` } : null;
},
async fetch(entry, ctx) {
const origin = resolveOrigin(entry);
if (!origin) throw new Error(`breezy: cannot derive API URL for ${entry.name}`);
const apiUrl = `${origin}/json`;
assertBreezyUrl(apiUrl);
// redirect:'error' + the host check above keep the final hostname pinned to
// the tenant — a server-side redirect can't bounce us off-domain (SSRF).
const json = /** @type {any} */ (await ctx.fetchJson(apiUrl, { redirect: 'error' }));
return parseBreezyResponse(json, entry.name);
},
};
/**
* Parse a Breezy `<tenant>.breezy.hr/json` response. Exported for unit tests.
*
* Breezy returns a top-level array of positions:
* [{ name, url, published_date?,
* location: { name?, city?, state?, country?: { name }, is_remote? } }]
*
* - url: Breezy supplies an absolute posting URL on the tenant domain
* (`https://<tenant>.breezy.hr/p/<id>-<slug>`); it is the Job contract's dedup
* key. The per-offer URL is display-only (recorded in the pipeline/history,
* never server-fetched here), so it is not host-locked — only a well-formed
* `https:` URL is required; rows without one are dropped.
* - location: prefer the ready-made `location.name`; else assemble from
* city / state / country.name, appending "Remote" when `is_remote` is truthy.
* - postedAt: parsed from the ISO `published_date` when present and valid — Breezy
* gives it for free in the list payload, so recency consumers can use it.
*
* @param {any} json
* @param {string} companyName
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
*/
export function parseBreezyResponse(json, companyName) {
const rows = Array.isArray(json) ? json : [];
const out = [];
for (const j of rows) {
if (!j || !j.name) continue;
// Resolve the posting URL (dedup key). Require a well-formed https: URL;
// a non-https or malformed/missing URL drops the row.
let url = '';
const rawUrl = typeof j.url === 'string' ? j.url : '';
if (rawUrl) {
try {
const parsed = new URL(rawUrl);
if (parsed.protocol === 'https:') url = parsed.href;
} catch { /* malformed → drop */ }
}
if (!url) continue;
const loc = j.location || {};
const remote = loc.is_remote ? 'Remote' : '';
const assembled = [loc.city, loc.state, loc.country?.name].filter(Boolean).join(', ');
const base = (typeof loc.name === 'string' && loc.name.trim()) ? loc.name.trim() : assembled;
const location = remote && !/remote/i.test(base)
? [base, remote].filter(Boolean).join(', ')
: base;
const job = {
title: String(j.name),
url,
company: companyName,
location,
};
const ts = Date.parse(j.published_date);
if (!Number.isNaN(ts)) job.postedAt = ts;
out.push(job);
}
return out;
}
+150
View File
@@ -0,0 +1,150 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Comeet (Spark Hire Recruit) provider — hits the public, no-auth careers API.
//
// The positions endpoint needs BOTH a company-uid and a per-tenant token:
// https://www.comeet.co/careers-api/2.0/company/<uid>/positions?token=<token>
// Neither is derivable from a branded careers URL, so unlike greenhouse/recruitee
// there is no slug→API shortcut: the full API URL must be supplied via the
// `api:` field (or pasted into `careers_url`). The API host is a single fixed
// origin, so the SSRF defence pins hostname to www.comeet.co AND requires the
// /careers-api/ path prefix (rather than a per-tenant subdomain regex).
const COMEET_API_HOST = 'www.comeet.co';
/** @param {unknown} raw */
function isComeetApiUrl(raw) {
if (typeof raw !== 'string' || !raw) return false;
let parsed;
try {
parsed = new URL(raw);
} catch {
return false;
}
return parsed.protocol === 'https:' && parsed.hostname === COMEET_API_HOST && parsed.pathname.startsWith('/careers-api/');
}
/** @param {string} url */
function assertComeetUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`comeet: invalid URL: ${redactToken(url)}`);
}
if (parsed.protocol !== 'https:') throw new Error(`comeet: URL must use HTTPS: ${redactToken(url)}`);
if (parsed.hostname !== COMEET_API_HOST)
throw new Error(`comeet: untrusted hostname "${parsed.hostname}" — must be ${COMEET_API_HOST}`);
if (!parsed.pathname.startsWith('/careers-api/'))
throw new Error(`comeet: URL path must be the careers-api endpoint: ${redactToken(url)}`);
return url;
}
// Redact the per-tenant ?token= so neither the (informational, possibly-logged)
// DetectHit url nor a thrown validation error carries the secret. Best-effort:
// falls back to a regex strip when the value can't be parsed as a URL.
function redactToken(url) {
try {
const parsed = new URL(url);
if (parsed.searchParams.has('token')) parsed.searchParams.set('token', 'REDACTED');
return parsed.href;
} catch {
return typeof url === 'string' ? url.replace(/([?&]token=)[^&#]*/gi, '$1REDACTED') : url;
}
}
/** @param {import('./_types.js').PortalEntry} entry */
function resolveApiUrl(entry) {
if (isComeetApiUrl(entry.api)) return entry.api;
// Fall back to careers_url only when it already IS the full careers-api URL
// (the branded www.comeet.com/jobs/... page carries no token, so it can't be
// turned into an API call).
if (isComeetApiUrl(entry.careers_url)) return entry.careers_url;
return null;
}
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
/** @type {Provider} */
export default {
id: 'comeet',
detect(entry) {
const apiUrl = resolveApiUrl(entry);
// The DetectHit url is informational (the framework may log it), so strip
// the secret ?token= before returning it — fetch() re-resolves the real
// URL from the entry, so redaction here is safe.
return apiUrl ? { url: redactToken(apiUrl) } : null;
},
async fetch(entry, ctx) {
const apiUrl = resolveApiUrl(entry);
if (!apiUrl) throw new Error(`comeet: cannot derive API URL for ${entry.name} (set api: to the full careers-api positions URL)`);
assertComeetUrl(apiUrl);
// redirect:'error' prevents SSRF via server-side redirects; combined with
// assertComeetUrl above it guarantees the final hostname stays www.comeet.co.
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
return parseComeetResponse(json, entry.name);
},
};
/**
* Parse a Comeet careers-api positions response. Exported for unit tests.
*
* Comeet returns a top-level ARRAY of position objects:
* [{ name, location: { name, is_remote }, url_active_page,
* url_comeet_hosted_page, time_updated, ... }]
*
* - url: prefer `url_active_page` (the tenant's live careers page), fall back to
* `url_comeet_hosted_page` (the Comeet-hosted page). Both are public, display-
* only URLs (recorded in the pipeline/history, never server-fetched here), so
* they are NOT host-locked. Require a well-formed https: URL; a position whose
* URL is missing/non-https/malformed is dropped (url is the dedup key).
* - location: `location.name`, appending "Remote" when `location.is_remote`.
*
* @param {any} json
* @param {string} companyName
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
*/
export function parseComeetResponse(json, companyName) {
const positions = Array.isArray(json) ? json : [];
return positions
.map(row => {
// Coerce each row to a safe object so null / non-object members can't throw.
const j = (row && typeof row === 'object') ? row : {};
// Resolve a display-only https URL; drop the position if none is usable.
let url = '';
const rawUrl = j.url_active_page || j.url_comeet_hosted_page || '';
if (typeof rawUrl === 'string' && rawUrl) {
try {
const parsed = new URL(rawUrl);
if (parsed.protocol === 'https:') url = parsed.href;
} catch {
// malformed URL → leave url = ''
}
}
const loc = j.location || {};
const remote = loc.is_remote ? 'Remote' : '';
const base = (typeof loc.name === 'string' && loc.name.trim()) ? loc.name.trim() : '';
// Append "Remote" only when the base location doesn't already say so.
const location = remote && !/remote/i.test(base)
? [base, remote].filter(Boolean).join(', ')
: base;
return {
title: (typeof j.name === 'string' ? j.name.trim() : ''),
url,
location,
company: companyName,
postedAt: toEpochMs(j.time_updated),
};
})
.filter(job => job.title && job.url);
}
+201
View File
@@ -0,0 +1,201 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Dassault Systèmes provider — hits the public Exalead "card search" API that
// powers www.3ds.com/careers/jobs. Single-company provider (like ibm.mjs): the
// endpoint is global to 3ds.com, so there's no per-tenant config.
//
// GET https://www.3ds.com/apisearch/card_search_api
// ?lang=en
// &r=f/card_content_type/career # only career cards
// &r=f/card_content_categories_facet/cards language/en # only the English copy
// &start={offset} # 0-based, 10 hits/page
//
// The response is Exalead XML (Content-Type: application/javascript), not JSON.
// We don't need a real XML parser — each posting is one <Hit>…</Hit> block whose
// fields are <Meta name="…"><MetaString name="value">…</MetaString> pairs.
//
// Why BOTH refinements matter (learned the hard way):
// * The `type=career` URL shorthand returns a *global* career-content index
// (~43k hits) where Dassault's own jobs are a small minority ranked first and
// the rest is third-party aggregated content (bcit.ca, dejobs.org, …). Using
// the proper `f/card_content_type/career` facet instead narrows to Dassault's
// own postings (5340 hits, every cta_1 URL on 3ds.com).
// * Without the language refinement each job is duplicated across ~12 languages.
// `cards language/en` collapses that to the 445 English postings.
// As a safety net we still keep only hits whose public URL is on *.3ds.com, so a
// future index change can't leak foreign postings back in.
//
// The Taleo apply backend (talentacquisition.3ds.com) is out of scope — we list
// from Exalead and link to the public 3ds.com job page.
const BASE = 'https://www.3ds.com/apisearch/card_search_api';
const REFINES = ['f/card_content_type/career', 'f/card_content_categories_facet/cards language/en'];
const PAGE_SIZE = 10; // Exalead returns 10 hits/page
const MAX_PAGES = 60; // safety cap on request count (~600 postings; the en set is ~445)
const MAX_JOBS = 1000; // cap total postings pulled
/** @param {number} start */
export function buildUrl(start) {
const p = new URLSearchParams();
p.set('lang', 'en'); // keeps facet labels (Country/City/…) in English
for (const r of REFINES) p.append('r', r);
p.set('start', String(start));
return `${BASE}?${p.toString()}`;
}
// Minimal HTML entity decoder — titles/URLs/locations carry named (&amp;, &apos;)
// and numeric (&#252; / &#xfc;) entities. Mirrors successfactors.mjs.
const NAMED_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
/** @param {string} s */
function decodeEntities(s) {
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (m, body) => {
if (body[0] === '#') {
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
}
return NAMED_ENTITIES[body.toLowerCase()] ?? m;
});
}
// Pull every <Meta name="X"><MetaString name="value">V</MetaString> pair from one
// <Hit> block into a Map. First value per name wins.
/** @param {string} hitXml @returns {Map<string,string>} */
function metaMap(hitXml) {
const map = new Map();
const re = /<Meta name="([^"]+)"[^>]*>\s*<MetaString[^>]*name="value"[^>]*>([\s\S]*?)<\/MetaString>/g;
let m;
while ((m = re.exec(hitXml)) !== null) {
if (!map.has(m[1])) map.set(m[1], m[2]);
}
return map;
}
// content_categories is a flat "Label/Value Label/Value …" string. Values can hold
// spaces and commas ("United Kingdom, Cambridge"), so we can't split on whitespace —
// we anchor on the known label tokens and slice each value up to the next label.
// Labels come back localized even at lang=en in some responses (the API has been
// seen emitting Spanish País/Ciudad), so we accept the common locale variants.
const LOC_LABELS = [
'Category', 'Type', 'Country', 'City', 'Products', 'Year', // en
'Categoría', 'Categoria', 'Tipo', 'País', 'Pais', 'Ciudad', 'Productos', 'Año', 'Ano', 'Área', 'Area', // es
'Catégorie', 'Pays', 'Ville', 'Produits', 'Année', 'Annee', // fr
];
const CITY_LABELS = new Set(['City', 'Ciudad', 'Ville']);
const COUNTRY_LABELS = new Set(['Country', 'País', 'Pais', 'Pays']);
const LABEL_RE = new RegExp('(^|\\s)(' + LOC_LABELS.join('|') + ')\\/', 'g');
/** @param {string} categories @returns {{city: string, country: string}} */
function parseCategories(categories) {
const marks = [];
let m;
LABEL_RE.lastIndex = 0;
while ((m = LABEL_RE.exec(categories)) !== null) {
marks.push({ label: m[2], keyStart: m.index + m[1].length, valStart: m.index + m[0].length });
}
let city = '';
let country = '';
for (let i = 0; i < marks.length; i++) {
const end = i + 1 < marks.length ? marks[i + 1].keyStart : categories.length;
const value = decodeEntities(categories.slice(marks[i].valStart, end)).trim();
if (!city && CITY_LABELS.has(marks[i].label)) city = value;
else if (!country && COUNTRY_LABELS.has(marks[i].label)) country = value;
}
return { city, country };
}
// "2026/07/03 18:22:13" → epoch ms (parsed as UTC so it's deterministic across
// machine timezones; the API doesn't state a zone, and consumers only use this
// for coarse recency ranking).
/** @param {string} ts @returns {number | undefined} */
function toEpochMs(ts) {
const m = /^(\d{4})\/(\d{2})\/(\d{2})[ T](\d{2}):(\d{2}):(\d{2})/.exec((ts || '').trim());
if (!m) return undefined;
const [, y, mo, d, h, mi, s] = m.map(Number);
const ms = Date.UTC(y, mo - 1, d, h, mi, s);
return Number.isFinite(ms) ? ms : undefined;
}
/**
* Parse one Exalead card-search XML page into normalized jobs.
* Deduped by card_id within the page; the fetch loop dedups again across pages.
* Keeps only real Dassault-hosted postings (public URL on *.3ds.com).
* @param {string} xml @param {string} entryName
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number, _id: string}>}
*/
export function parseHits(xml, entryName) {
const hits = typeof xml === 'string' ? xml.match(/<Hit\b[\s\S]*?<\/Hit>/g) : null;
if (!hits) return [];
const byId = new Map();
for (const hit of hits) {
const meta = metaMap(hit);
const title = decodeEntities((meta.get('content_title') || '').replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
const url = decodeEntities((meta.get('content_cta_1_url') || '').trim());
if (!title || !url) continue;
// Safety net: only accept postings hosted on 3ds.com. The refined query
// already guarantees this, but a broadened index must never leak through.
let host;
try {
host = new URL(url).host.toLowerCase();
} catch {
continue;
}
if (host !== '3ds.com' && !host.endsWith('.3ds.com')) continue;
const { city, country } = parseCategories(meta.get('content_categories') || '');
const postedAt = toEpochMs(meta.get('content_start_datetime') || meta.get('card_update_timestamp') || '');
// card_id is the natural dedup key; fall back to the URL when absent.
const id = (meta.get('card_id') || '').trim() || url;
if (byId.has(id)) continue;
const job = { title, url, company: entryName || 'Dassault Systèmes', location: city || country, _id: id };
if (postedAt !== undefined) job.postedAt = postedAt;
byId.set(id, job);
}
return [...byId.values()];
}
/** @type {Provider} */
export default {
id: 'dassault',
detect(entry) {
const url = entry.api || entry.careers_url || '';
if (typeof url !== 'string') return null;
// Match the host, not a path segment, so evil.com/x.3ds.com can't spoof it.
try {
const host = new URL(url).host.toLowerCase();
if (host === '3ds.com' || host.endsWith('.3ds.com')) return { url };
} catch {
/* not an absolute URL */
}
return null;
},
async fetch(entry, ctx) {
const jobs = [];
const seen = new Set();
for (let page = 0; page < MAX_PAGES; page++) {
const xml = await ctx.fetchText(buildUrl(page * PAGE_SIZE), {
redirect: 'error',
headers: { accept: 'application/xml, text/xml, */*' },
});
const parsed = parseHits(xml, entry.name);
if (parsed.length === 0) break; // past the last page
let fresh = 0;
for (const job of parsed) {
if (seen.has(job._id)) continue;
seen.add(job._id);
fresh++;
const { _id, ...clean } = job; // drop the internal dedup key
jobs.push(clean);
}
if (fresh === 0) break; // server ignored the offset / looped
if (jobs.length >= MAX_JOBS) break;
}
return jobs;
},
};
+134
View File
@@ -0,0 +1,134 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Get on Board provider — board-wide feed for the tech "programming" category
// (https://www.getonbrd.com/api/v0/categories/programming/jobs). Public,
// zero-auth JSON:API. `expand[]=company` embeds the company so its name is
// available at the list level. The broad category feed is fetched (not the
// server-side ?query= search, which requires a query and narrows results) so
// scan.mjs's title_filter can gate on the configured titles instead. Pages are
// fetched until one comes back short/empty or the page cap is reached (default
// 3, override with `max_pages` on the portal entry).
//
// Wire in via a `job_boards:` entry with `provider: getonbrd`.
const FEED_BASE = 'https://www.getonbrd.com/api/v0/categories/programming/jobs';
const TRUSTED_HOST = 'www.getonbrd.com';
const PER_PAGE = 100;
const DEFAULT_MAX_PAGES = 3;
const MAX_PAGES_CAP = 50;
/** @param {string} url */
function assertGetonbrdUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`getonbrd: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`getonbrd: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST) {
throw new Error(`getonbrd: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
}
return url;
}
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
function resolveMaxPages(entry) {
const v = entry?.max_pages;
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
return DEFAULT_MAX_PAGES;
}
/**
* Normalize a single Get on Board job (JSON:API resource). Exported for tests.
*
* Field mapping → the normalized Job shape:
* - title: `attributes.title`, trimmed (items without one are dropped).
* - url: `links.public_url` — an absolute `https:` posting URL host-locked
* to www.getonbrd.com (off-host or non-https drops the item). It is
* the dedup key and is display-only (never server-fetched here).
* - company: `attributes.company.data.attributes.name` (from `expand[]=company`),
* falling back to the portal entry name, then "Get on Board".
* - location: "Remote" when `attributes.remote` is true, else the joined
* `attributes.countries` (an array of country names in the live
* API; a plain string is tolerated for older payloads).
* - postedAt: `attributes.published_at` (epoch SECONDS) → epoch ms (omitted when absent).
*
* @param {any} j
* @param {string} [fallbackCompany]
* @returns {{ title: string, url: string, company: string, location: string, postedAt?: number } | null}
*/
export function normalizeGetonbrdJob(j, fallbackCompany) {
if (!j || typeof j !== 'object' || !j.attributes || typeof j.attributes !== 'object') return null;
const attr = j.attributes;
const title = typeof attr.title === 'string' ? attr.title.trim() : '';
if (!title) return null;
// url must be an absolute https posting link on www.getonbrd.com.
let url = '';
const rawUrl = j.links && typeof j.links.public_url === 'string' ? j.links.public_url.trim() : '';
if (rawUrl) {
try {
const parsed = new URL(rawUrl);
if (parsed.protocol === 'https:' && parsed.hostname === TRUSTED_HOST) url = parsed.href;
} catch {
// malformed URL → leave url = '' → dropped below
}
}
if (!url) return null;
const name = attr.company?.data?.attributes?.name;
const company =
typeof name === 'string' && name.trim() ? name.trim() : fallbackCompany || 'Get on Board';
// `attributes.countries` is an array of country names in the live API (e.g.
// ["Chile"]); older/edge payloads may send a plain string. Handle both.
let location = '';
if (attr.remote === true) {
location = 'Remote';
} else if (Array.isArray(attr.countries)) {
location = attr.countries
.filter((c) => typeof c === 'string' && c.trim())
.map((c) => c.trim())
.join(', ');
} else if (typeof attr.countries === 'string') {
location = attr.countries.trim();
}
/** @type {{ title: string, url: string, company: string, location: string, postedAt?: number }} */
const job = { title, url, company, location };
// `attributes.published_at` is epoch SECONDS → convert to ms (omitted when absent).
if (Number.isFinite(attr.published_at) && attr.published_at > 0) job.postedAt = attr.published_at * 1000;
return job;
}
/** @type {Provider} */
export default {
id: 'getonbrd',
async fetch(entry, ctx) {
assertGetonbrdUrl(FEED_BASE);
const maxPages = resolveMaxPages(entry);
const fallbackCompany = entry?.name;
const out = [];
for (let page = 1; page <= maxPages; page++) {
const url = `${FEED_BASE}?per_page=${PER_PAGE}&expand[]=company&page=${page}`;
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(url, { redirect: 'error' });
if (!json || !Array.isArray(json.data)) {
throw new Error(
`getonbrd: unexpected API response on page ${page} — expected { data: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`,
);
}
for (const j of json.data) {
const normalized = normalizeGetonbrdJob(j, fallbackCompany);
if (normalized) out.push(normalized);
}
if (json.data.length < PER_PAGE) break; // short page → last page reached
}
return out;
},
};
+248
View File
@@ -0,0 +1,248 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Glints provider — hits the public GraphQL v2-alc endpoint.
//
// Glints (glints.com) covers Singapore, Indonesia, Malaysia, and Vietnam.
// Their internal API is a no-auth GraphQL endpoint at /api/v2-alc/graphql that
// powers the job search page. The schema is reverse-engineered and may change.
//
// This provider is designed for explicit `provider: glints` in portals.yml.
// Auto-detection is not supported — Glints is a job board aggregator, not
// a company ATS.
//
// Portal entry fields (all optional except `provider`):
// api — GraphQL endpoint URL (default: https://glints.com/api/v2-alc/graphql)
// searchKeywords — Search keywords string (default: '')
// countryCode — Two-letter country code (default: "ID" for Indonesia)
// pageSize — Results per page (default: 30)
// maxPages — Maximum pages (default: 3)
// graphqlQuery — Custom GraphQL query string. If not provided, the
// built-in default query is used.
const DEFAULT_API = 'https://glints.com/api/v2-alc/graphql';
const DEFAULT_COUNTRY = 'ID';
const DEFAULT_PAGE_SIZE = 30;
const DEFAULT_MAX_PAGES = 3;
const ALLOWED_GLINTS_HOSTS = new Set([
'glints.com',
'www.glints.com',
'glints.id',
]);
// Default GraphQL query — reverse-engineered from glints.com/id search.
// Uses the searchJobsV3 operation which replaced the older opportunities query.
const DEFAULT_GRAPHQL_QUERY = `
query searchJobsV3($data: JobSearchConditionInput!) {
searchJobsV3(data: $data) {
jobsInPage {
id
title
company {
name
brandName
}
city {
name
}
country {
code
name
}
salaries {
salaryType
salaryMode
maxAmount
minAmount
CurrencyCode
}
createdAt
}
expInfo
hasMore
}
}`;
/** @param {string} url */
function assertGlintsUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`glints: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`glints: URL must use HTTPS: ${url}`);
if (!ALLOWED_GLINTS_HOSTS.has(parsed.hostname))
throw new Error(`glints: untrusted hostname "${parsed.hostname}" — must be one of: ${[...ALLOWED_GLINTS_HOSTS].join(', ')}`);
return url;
}
// NaN-safe Date.parse
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
/**
* Derive the job detail base URL from the API hostname.
* @param {string} apiUrl
* @returns {string}
*/
function deriveBaseUrl(apiUrl) {
try {
const parsed = new URL(apiUrl);
return `${parsed.protocol}//${parsed.hostname}`;
} catch {
return 'https://glints.com';
}
}
/**
* Parse a single Glints job (searchJobsV3 response) into the canonical Job shape.
*
* This parser is exported as a named export for unit tests.
*
* @param {any} item — raw GraphQL result item from searchJobsV3
* @param {string} baseUrl — scheme + hostname for resolving relative URLs
* @param {string} fallbackCompany — company name fallback from portal entry
* @returns {{title: string, url: string, company: string, location: string, postedAt: number|undefined}|null}
*/
export function parseGlintsItem(item, baseUrl, fallbackCompany) {
if (!item || typeof item !== 'object') return null;
const title = (item.title || '').trim();
if (!title) return null;
// Build job URL from the job ID
const jobId = (item.id || '').trim();
if (!jobId) return null;
const url = `${baseUrl}/id/opportunities/jobs/${jobId}`;
// Validate URL hostname
try {
const parsed = new URL(url);
const hostname = parsed.hostname;
const allowed = ALLOWED_GLINTS_HOSTS.has(hostname) || hostname.endsWith('.glints.com');
if (!allowed) return null;
} catch {
return null;
}
const company = (item.company?.name || item.company?.brandName || fallbackCompany || '').trim();
const location = (item.city?.name || '').trim();
const postedAt = toEpochMs(item.createdAt);
return { title, url, company, location, ...(postedAt != null ? { postedAt } : {}) };
}
/**
* Execute a single GraphQL query page.
* @param {string} apiUrl
* @param {string} query
* @param {object} variables
* @param {import('./_types.js').Context} ctx
* @returns {Promise<any>}
*/
async function graphqlPage(apiUrl, query, variables, ctx) {
const body = JSON.stringify({ operationName: 'searchJobsV3', query, variables });
// Glints firewall blocks non-browser User-Agents, so use a real Chrome UA
try {
const res = await ctx.fetchJson(apiUrl, {
method: 'POST',
headers: {
'content-type': 'application/json',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'origin': 'https://glints.com',
'referer': 'https://glints.com/id/opportunities/jobs/explore',
},
body,
redirect: 'error',
});
return res;
} catch (err) {
// On POST, some servers return non-JSON errors; attempt text fallback
if (err.status && err.body) {
let detail = '';
try {
const parsed = JSON.parse(err.body);
detail = parsed.errors?.[0]?.message || err.body.slice(0, 200);
} catch {
detail = err.body.slice(0, 200);
}
throw new Error(`glints: HTTP ${err.status} — ${detail}`);
}
throw err;
}
}
/** @type {Provider} */
export default {
id: 'glints',
detect(_entry) {
// Glints is a job board aggregator, not a company ATS.
// Auto-detection is intentionally not supported —
// use `provider: glints` explicitly in portals.yml.
return null;
},
async fetch(entry, ctx) {
const apiUrl = entry.api || DEFAULT_API;
assertGlintsUrl(apiUrl);
const baseUrl = deriveBaseUrl(apiUrl);
const query = entry.graphqlQuery || DEFAULT_GRAPHQL_QUERY;
const keywords = entry.searchKeywords || '';
const country = entry.countryCode || DEFAULT_COUNTRY;
const pageSize = Number(entry.pageSize) || DEFAULT_PAGE_SIZE;
const maxPages = Number(entry.maxPages) || DEFAULT_MAX_PAGES;
const fallbackCompany = entry.name || '';
const allJobs = [];
for (let page = 1; page <= maxPages; page++) {
const variables = {
data: {
SearchTerm: keywords,
CountryCode: country,
includeExternalJobs: true,
pageSize: pageSize,
page: page,
},
};
let json;
try {
json = /** @type {any} */ (await graphqlPage(apiUrl, query, variables, ctx));
} catch (err) {
if (page === 1) throw err;
console.error(`glints: page ${page} fetch failed — ${err.message}`);
break;
}
const jobsInPage = json?.data?.searchJobsV3?.jobsInPage;
if (!Array.isArray(jobsInPage)) {
if (page === 1) throw new Error(`glints: unexpected API response — ${JSON.stringify(json).slice(0, 200)}`);
break;
}
if (jobsInPage.length === 0) break;
for (const item of jobsInPage) {
const job = parseGlintsItem(item, baseUrl, fallbackCompany);
if (job) allJobs.push(job);
}
// Stop if no more pages
if (json?.data?.searchJobsV3?.hasMore === false) break;
if (jobsInPage.length < pageSize) break;
// Rate-limit courtesy delay
await new Promise(resolve => setTimeout(resolve, 300));
}
return allJobs;
},
};
+76
View File
@@ -0,0 +1,76 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Greenhouse provider — hits the public boards-api JSON endpoint.
// Handles both explicit `api:` URLs and auto-detection from `careers_url`.
const ALLOWED_GREENHOUSE_HOSTS = new Set([
'boards-api.greenhouse.io',
'boards.greenhouse.io',
'job-boards.greenhouse.io',
'job-boards.eu.greenhouse.io',
]);
/** @param {string} url */
function assertGreenhouseUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`greenhouse: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`greenhouse: URL must use HTTPS: ${url}`);
if (!ALLOWED_GREENHOUSE_HOSTS.has(parsed.hostname))
throw new Error(`greenhouse: untrusted hostname "${parsed.hostname}" — must be one of: ${[...ALLOWED_GREENHOUSE_HOSTS].join(', ')}`);
return url;
}
/** @param {import('./_types.js').PortalEntry} entry */
function resolveApiUrl(entry) {
if (entry.api) {
assertGreenhouseUrl(entry.api);
return entry.api;
}
const url = entry.careers_url || '';
const match = url.match(/job-boards(?:\.eu)?\.greenhouse\.io\/([^/?#]+)/);
if (match) return `https://boards-api.greenhouse.io/v1/boards/${match[1]}/jobs`;
return null;
}
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
/** @type {Provider} */
export default {
id: 'greenhouse',
detect(entry) {
try {
const apiUrl = resolveApiUrl(entry);
return apiUrl ? { url: apiUrl } : null;
} catch {
return null;
}
},
async fetch(entry, ctx) {
const apiUrl = resolveApiUrl(entry);
if (!apiUrl) throw new Error(`greenhouse: cannot derive API URL for ${entry.name}`);
assertGreenhouseUrl(apiUrl);
// redirect:'error' prevents SSRF via server-side redirects; combined with
// assertGreenhouseUrl above it guarantees the final hostname stays in the allowlist.
const json = /** @type {any} */ (await ctx.fetchJson(apiUrl, { redirect: 'error' }));
const jobs = Array.isArray(json?.jobs) ? json.jobs : [];
return jobs.filter(/** @param {any} j */ j => j.absolute_url).map(/** @param {any} j */ j => ({
title: j.title || '',
url: j.absolute_url,
company: entry.name,
location: j.location?.name || '',
postedAt: toEpochMs(j.first_published),
}));
},
};
+182
View File
@@ -0,0 +1,182 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Hacker News "Ask HN: Who is hiring?" provider — no auth required.
//
// Algorithm:
// 1. Find the current monthly hiring thread via the Algolia HN search API
// (search_by_date, tags=story, query="Ask HN Who is hiring").
// 2. Fetch the thread's item from the Algolia items API; top-level `children`
// are individual job posts left as top-level comments.
// 3. Parse each comment: the first non-empty line is treated as the title/header
// (many follow "Company | Role | Location | URL" but this is free-form, so
// we extract defensively — a URL is pulled out wherever it appears; the
// first line becomes the title; company/location are left empty when the
// format doesn't match the pipe-delimited convention).
//
// Wire in via a `job_boards:` entry with `provider: hackernews`.
const SEARCH_URL =
'https://hn.algolia.com/api/v1/search_by_date?tags=story&query=Ask%20HN%20Who%20is%20hiring&hitsPerPage=5';
/** @param {string} id */
function itemUrl(id) {
return `https://hn.algolia.com/api/v1/items/${id}`;
}
/**
* Scan raw text for the first absolute http/https URL. Returns '' if none found.
* @param {string} text
*/
function extractUrl(text) {
const m = text.match(/https?:\/\/[^\s<>"')]+/);
return m ? m[0].replace(/[.,;!?)]+$/, '') : '';
}
// Named HTML entities we decode in comment bodies. Kept in single map
/** @type {Record<string, string>} */
const ENTITY_MAP = {
'&amp;': '&',
'&lt;': '<',
'&gt;': '>',
'&quot;': '"',
'&#x27;': "'",
'&#39;': "'",
'&nbsp;': ' ',
};
const ENTITY_RE = /&amp;|&lt;|&gt;|&quot;|&#x27;|&#39;|&nbsp;/g;
/**
* Parse a single HN comment text into a normalized job object.
* The canonical format is "Company | Role | Location | URL" on the first line,
* but posts are free-form — we only guarantee title (first line) and url (first
* URL found anywhere in the text). company and location are extracted from the
* pipe-delimited header when present; left empty otherwise.
*
* Exported for unit testing.
*
* @param {string} text Raw comment text (may contain HTML; tags are stripped).
* @param {string} threadUrl Fallback url (the HN thread) if no URL in comment.
* @returns {{ title: string, url: string, company: string, location: string } | null}
* null when the comment is empty, deleted, or carries no usable title.
*/
export function parseHnComment(text, threadUrl = '') {
if (!text || typeof text !== 'string') return null;
// Strip HTML. Anchors: keep the href value in place so URL extraction works.
// Block-level tags (<p>, <br>, <div>, <li>, headings) become newlines so that
// body paragraphs never bleed into the first header line after the join.
const plain = text
.replace(/<a\s[^>]*href="([^"]+)"[^>]*>.*?<\/a>/gi, (_, href) => href)
.replace(/<\/?(?:p|br|div|li|h[1-6])\b[^>]*>/gi, '\n')
.replace(/<[^>]+>/g, ' ')
.replace(ENTITY_RE, (m) => ENTITY_MAP[m])
.replace(/\r\n/g, '\n')
.replace(/\r/g, '\n');
// Split into lines; first non-blank line is the header.
const lines = plain.split('\n').map(l => l.trim()).filter(Boolean);
if (lines.length === 0) return null;
const firstLine = lines[0];
if (!firstLine) return null;
// Try to parse pipe-delimited header: Company | Role | Location | URL (4+ parts)
// or Company | Role | Location (3 parts).
const parts = firstLine.split('|').map(p => p.trim());
let company = '';
let location = '';
if (parts.length >= 3) {
company = parts[0];
// Role is parts[1] — used in title construction below, not stored separately.
// Strip any embedded URL from the location field (e.g. when the URL is run
// into the same pipe segment rather than given its own 4th segment).
location = parts[2].replace(/https?:\/\/\S+/g, '').trim();
} else if (parts.length === 2) {
company = parts[0];
}
// Single-part first line: title only, company/location stay empty.
// Build a clean title from the first line (strip any embedded URL from it).
const title = firstLine.replace(/https?:\/\/[^\s]+/g, '').replace(/\s{2,}/g, ' ').trim();
if (!title) return null;
// Find first URL anywhere in the full text.
const url = extractUrl(plain) || threadUrl;
if (!url) return null;
return { title, url, company, location };
}
/**
* Find the objectID of the latest "Ask HN: Who is hiring?" story.
* Returns null when the search yields no matching hits.
* @param {unknown} data Parsed Algolia search response.
*/
export function resolveLatestThreadId(data) {
if (!data || typeof data !== 'object') return null;
const hits = /** @type {any} */ (data).hits;
if (!Array.isArray(hits) || hits.length === 0) return null;
// Algolia search_by_date returns newest first; find the first hit whose title
// matches the monthly thread pattern (case-insensitive).
const RE = /ask\s+hn[:\s]+who\s+is\s+hiring/i;
for (const hit of hits) {
if (hit && typeof hit.objectID === 'string' && typeof hit.title === 'string') {
if (RE.test(hit.title)) return hit.objectID;
}
}
return null;
}
/** @type {Provider} */
export default {
id: 'hackernews',
async fetch(entry, ctx) {
// Step 1: Find the latest "Who is hiring?" story id.
const searchData = await ctx.fetchJson(SEARCH_URL, { redirect: 'error' });
const threadId = resolveLatestThreadId(searchData);
if (!threadId) {
throw new Error('hackernews: could not find "Ask HN: Who is hiring?" thread in search results');
}
const threadHnUrl = `https://news.ycombinator.com/item?id=${threadId}`;
// Step 2: Fetch the thread item (children = top-level job comments).
const item = await ctx.fetchJson(itemUrl(threadId), { redirect: 'error' });
if (!item || typeof item !== 'object') {
throw new Error(`hackernews: unexpected item response for thread ${threadId}`);
}
const children = /** @type {any} */ (item).children;
if (!Array.isArray(children)) return [];
// Step 3: Parse each comment.
const jobs = [];
for (const child of children) {
// Skip deleted / dead / empty comments.
if (!child || child.deleted || child.dead) continue;
const text = typeof child.text === 'string' ? child.text : '';
if (!text.trim()) continue;
const parsed = parseHnComment(text, threadHnUrl);
if (!parsed) continue;
jobs.push({
title: parsed.title,
url: parsed.url,
company: parsed.company || (entry.name || 'HN Hiring'),
location: parsed.location,
// Algolia returns created_at as ISO string.
...(child.created_at
? { postedAt: Date.parse(child.created_at) || undefined }
: {}),
});
}
return jobs;
},
};
+152
View File
@@ -0,0 +1,152 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// HigherEdJobs.com RSS category feed provider.
// (https://www.higheredjobs.com/rss/categoryFeed.cfm?catID=68). The feed is
// public, no-auth, and XML, so it is parsed in-process with the same tiny tag
// extractor approach as providers/weworkremotely.mjs rather than adding an
// XML dependency.
//
// Wire in via a `job_boards:` entry with `provider: higheredjobs`.
const DEFAULT_CAT_ID = 68; // Higher Education category
const TRUSTED_HOST = 'www.higheredjobs.com';
function feedUrlFor(catId) {
const catID = typeof catId === 'number' && Number.isFinite(catId) ? catId : DEFAULT_CAT_ID;
return `https://www.higheredjobs.com/rss/categoryFeed.cfm?catID=${catID}`;
}
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
function fallbackCompany(entry) {
return typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : 'HigherEdJobs';
}
/** @type {Provider} */
export default {
id: 'higheredjobs',
detect(entry) {
if (entry?.provider !== 'higheredjobs') return null;
return { url: feedUrlFor(entry.cat_id) };
},
async fetch(entry, ctx) {
const feedUrl = feedUrlFor(entry?.cat_id);
// redirect:'error' prevents SSRF via server-side redirects; combined with
// cleanUrl below it keeps the request pinned to www.higheredjobs.com.
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
return parseHigherEdJobsFeed(text, fallbackCompany(entry));
},
};
function fromCodePoint(cp) {
try {
return String.fromCodePoint(cp);
} catch {
return '';
}
}
// Decode the XML entities that appear in RSS text: numeric (&#38; / &#x27;)
// and the named five. Numeric forms are decoded first; &amp; is decoded LAST
// so a literal "&amp;lt;" yields "&lt;" rather than over-decoding to "<".
function decodeXmlEntities(s) {
return s
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, '&');
}
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
function extractText(inner) {
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
if (cdata) return cdata[1].trim();
return decodeXmlEntities(inner).trim();
}
// Extract the text of the first <tag>...</tag> in a block. Returns '' when absent.
function tagText(block, tag) {
const m = block.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, 'i'));
return m ? extractText(m[1]) : '';
}
function cleanUrl(value) {
if (!value) return '';
const trimmed = value.trim();
try {
const parsed = new URL(trimmed);
const host = parsed.hostname.toLowerCase();
// Exact-match host (no subdomain wildcard): higheredjobs serves only
// www.higheredjobs.com for posting URLs.
const trusted = host === TRUSTED_HOST;
return parsed.protocol === 'https:' && trusted ? parsed.href : '';
} catch {
return '';
}
}
// HigherEdJobs <description> is "Institution Name (City, ST)". Split on the
// last " (" to separate company from location. When no parens are present the
// whole description is treated as the company and location is empty.
function splitDescription(rawDescription, defaultCompany) {
const text = rawDescription.trim();
const open = text.lastIndexOf(' (');
if (open > 0) {
const close = text.lastIndexOf(')');
const company = text.slice(0, open).trim();
const location = close > open ? text.slice(open + 2, close).trim() : text.slice(open + 2).trim();
if (company) return { company, location };
}
return { company: text || defaultCompany, location: '' };
}
/**
* Parse HigherEdJobs' public RSS category feed. Exported for unit tests.
*
* Shape: `<rss><channel><item>...</item>...</channel></rss>`, each item
* carrying `<title>` (plain job title), `<description>` ("Institution (City,
* ST)"), `<link>`, `<pubDate>`, and `<guid>`. Company and location are
* derived from <description>. The RSS <link> is the dedup key; items without
* a usable absolute URL on www.higheredjobs.com are dropped.
*
* @param {string} xml - raw RSS feed body
* @param {string} [defaultCompany] - fallback company for empty descriptions
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
*/
export function parseHigherEdJobsFeed(xml, defaultCompany = 'HigherEdJobs') {
if (typeof xml !== 'string') return [];
const fallback = typeof defaultCompany === 'string' && defaultCompany.trim() ? defaultCompany.trim() : 'HigherEdJobs';
const jobs = [];
const blocks = xml.match(/<item\b[^>]*>[\s\S]*?<\/item>/gi) || [];
for (const item of blocks) {
const url = cleanUrl(tagText(item, 'link'));
if (!url) continue;
const title = tagText(item, 'title');
if (!title) continue;
const { company, location } = splitDescription(tagText(item, 'description'), fallback);
jobs.push({
title,
company,
location,
url,
postedAt: toEpochMs(tagText(item, 'pubDate')),
});
}
return jobs;
}
+129
View File
@@ -0,0 +1,129 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Himalayas provider - board-wide remote jobs API
// (https://himalayas.app/jobs/api?limit=50). Returns { jobs: [...] }. The
// full feed is fetched so scan.mjs's title_filter / location_filter can do
// the local gating consistently with other zero-token board providers.
//
// Wire in via a `job_boards:` entry with `provider: himalayas`.
const FEED_URL = 'https://himalayas.app/jobs/api?limit=50';
const TRUSTED_HOST = 'himalayas.app';
/** @param {string} url */
function assertHimalayasUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`himalayas: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`himalayas: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST) {
throw new Error(`himalayas: untrusted hostname "${parsed.hostname}" - must be ${TRUSTED_HOST}`);
}
return url;
}
function cleanText(value) {
return typeof value === 'string' ? value.trim() : '';
}
function cleanHimalayasUrl(value) {
const raw = cleanText(value);
if (!raw) return '';
try {
const parsed = new URL(raw);
const host = parsed.hostname.toLowerCase();
const trusted = host === TRUSTED_HOST || host.endsWith(`.${TRUSTED_HOST}`);
return parsed.protocol === 'https:' && trusted ? parsed.href : '';
} catch {
return '';
}
}
function locationText(value) {
if (!Array.isArray(value)) return '';
return value
.filter(v => typeof v === 'string' && v.trim())
.map(v => v.trim())
.join(', ');
}
// Himalayas pubDate is currently epoch seconds. Accept milliseconds and
// parseable date strings too so the parser survives small API shape changes.
function toEpochMs(value) {
if (typeof value === 'number' && Number.isFinite(value)) {
return value < 1_000_000_000_000 ? value * 1000 : value;
}
if (typeof value === 'string' && value.trim()) {
const numeric = Number(value);
if (Number.isFinite(numeric)) return numeric < 1_000_000_000_000 ? numeric * 1000 : numeric;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
return undefined;
}
/** @type {Provider} */
export default {
id: 'himalayas',
detect(entry) {
return entry?.provider === 'himalayas' ? { url: FEED_URL } : null;
},
/**
* Fetches and normalizes postings from the Himalayas public feed.
* @param {{ provider?: string }} entry - The job_boards entry being processed.
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
* @returns {Promise<Array<{title: string, url: string, company: string, location: string, postedAt?: number}>>}
*/
async fetch(entry, ctx) {
const feedUrl = assertHimalayasUrl(FEED_URL);
// redirect:'error' prevents SSRF via server-side redirects; combined with
// assertHimalayasUrl above it keeps the request pinned to himalayas.app.
const json = await ctx.fetchJson(feedUrl, { redirect: 'error' });
if (!json || !Array.isArray(json.jobs)) {
throw new Error(`himalayas: unexpected API response - expected { jobs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
}
return parseHimalayasResponse(json);
},
};
/**
* Parse Himalayas' public jobs API response. Exported for unit tests.
*
* Shape: `{ jobs: [...] }`, where each job currently carries `title`,
* `companyName`, `locationRestrictions`, `applicationLink`, `guid`,
* `pubDate`, and `companySlug`. `applicationLink` is preferred over `guid`
* and used as the dedup key after HTTPS + host validation.
*
* @param {unknown} json - raw parsed API response
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
*/
export function parseHimalayasResponse(json) {
if (!json || typeof json !== 'object' || !Array.isArray(json.jobs)) return [];
const jobs = [];
for (const item of json.jobs) {
if (!item || typeof item !== 'object') continue;
const title = cleanText(item.title);
if (!title) continue;
const url = cleanHimalayasUrl(item.applicationLink) || cleanHimalayasUrl(item.guid);
if (!url) continue;
jobs.push({
title,
url,
company: cleanText(item.companyName),
location: locationText(item.locationRestrictions),
postedAt: toEpochMs(item.pubDate),
});
}
return jobs;
}
+124
View File
@@ -0,0 +1,124 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// IBM careers provider — POSTs to IBM's careers search API (Elasticsearch-style;
// the same endpoint the ibm.com/careers/search UI calls). One endpoint serves
// every locale, so results are language-agnostic (lang: "zz").
//
// Configure via a `job_boards` (or `tracked_companies`) entry with `provider: ibm`
// and an optional `ibm:` block of facet filters:
//
// - name: IBM Germany — SWE & Data
// provider: ibm
// ibm:
// country: Germany # field_keyword_05
// categories: ["Software Engineering", "Data & Analytics"] # field_keyword_08
// enabled: true
//
// Omit `country` / `categories` to widen the search. Paginates via `from` until
// the result set is exhausted or MAX_RECORDS is reached.
const API_URL = 'https://www-api.ibm.com/search/api/v2';
const PAGE_SIZE = 30;
const MAX_RECORDS = 600; // safety cap on pagination
/**
* Builds the Elasticsearch post_filter from the entry's `ibm:` config.
* @param {{ country?: string, categories?: string[] }} cfg
*/
export function buildPostFilter(cfg) {
const must = [];
// Sanitize operator config: keep only non-empty trimmed strings so a stray/
// mistyped entry can't inject empty or non-string filter terms.
const categories = Array.isArray(cfg.categories)
? cfg.categories.filter(c => typeof c === 'string' && c.trim()).map(c => c.trim())
: [];
if (categories.length) {
must.push({ bool: { should: categories.map(c => ({ term: { field_keyword_08: c } })) } });
}
const country = typeof cfg.country === 'string' ? cfg.country.trim() : '';
if (country) must.push({ term: { field_keyword_05: country } });
return { bool: { must } };
}
/**
* Normalizes one page of the IBM careers API response into job entries.
* Throws if the response doesn't carry the expected `hits.hits[]` shape, so a
* silent endpoint change surfaces as a hard error instead of empty results.
* @param {any} json - A single API response page.
* @returns {Array<{title: string, url: string, company: string, location: string}>}
*/
export function parseIbmResponse(json) {
const hits = json && json.hits && Array.isArray(json.hits.hits) ? json.hits.hits : null;
if (!hits) {
throw new Error(`ibm: unexpected API response — expected hits.hits[], got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
}
const out = [];
for (const h of hits) {
const s = (h && h._source) || {};
if (typeof s.title !== 'string' || s.title.trim() === '') continue;
if (typeof s.url !== 'string' || !/^https?:\/\//i.test(s.url.trim())) continue;
const loc = typeof s.field_keyword_19 === 'string' ? s.field_keyword_19.trim() : '';
const mode = typeof s.field_keyword_17 === 'string' ? s.field_keyword_17.trim() : '';
out.push({
title: s.title.trim(),
url: s.url.trim(),
company: 'IBM',
location: [loc, mode].filter(Boolean).join(' · '),
});
}
return out;
}
/** @type {Provider} */
export default {
id: 'ibm',
/**
* Fetches and normalizes postings from IBM's careers search API.
* @param {{ name?: string, ibm?: { country?: string, categories?: string[] } }} entry
* @param {{ fetchJson: (url: string, opts?: object) => Promise<any> }} ctx
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>}
*/
async fetch(entry, ctx) {
const postFilter = buildPostFilter(entry.ibm || {});
/** @type {Array<{title: string, url: string, company: string, location: string}>} */
const out = [];
for (let from = 0; from < MAX_RECORDS; from += PAGE_SIZE) {
const body = {
appId: 'careers',
scopes: ['careers2'],
query: { bool: { must: [] } },
post_filter: postFilter,
size: PAGE_SIZE,
from,
sort: [{ _score: 'desc' }, { pageviews: 'desc' }],
lang: 'zz',
localeSelector: {},
sm: { query: '', lang: 'zz' },
_source: ['_id', 'title', 'url', 'description', 'language',
'field_keyword_05', 'field_keyword_08', 'field_keyword_17',
'field_keyword_18', 'field_keyword_19'],
};
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(API_URL, {
method: 'POST',
headers: { 'content-type': 'application/json', accept: 'application/json' },
body: JSON.stringify(body),
redirect: 'error',
});
out.push(...parseIbmResponse(json));
// Stop on the first short page. IBM's `total` is unreliable (often
// capped at the page size), so don't trust it for the loop bound —
// MAX_RECORDS is the hard ceiling.
if (json.hits.hits.length < PAGE_SIZE) break;
}
return out;
},
};
+142
View File
@@ -0,0 +1,142 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// JibeApply provider — hits the /api/jobs endpoint on the same hostname.
// Auto-detects from careers_url pattern `https://<slug>.jibeapply.com`.
// iCIMS acquired Jibe in 2019; some tenants run it on a branded custom
// domain instead. The JSON API at /api/jobs is stable across all tenants.
// Safety cap on pagination — applied regardless of what the upstream reports
// as totalCount, so a misbehaving/compromised API can't drive this into
// fetching thousands of pages. Every known real tenant (10/page) stays well
// under the default of 50 pages (500 jobs); override with `max_pages` on the
// portal entry for a tenant that genuinely exceeds it — each page is a
// sequential round-trip, so raising the default for everyone isn't free.
const DEFAULT_MAX_PAGES = 50;
const MAX_PAGES_CAP = 500;
// Fallback page size when a response carries no jobs and no usable `count`
// (e.g. an empty first page). Confirmed live against real tenants — JibeApply
// reports this as `filter.displayLimit` and it has been 10 on every tenant
// observed so far.
const DEFAULT_PAGE_SIZE = 10;
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
function resolveMaxPages(entry) {
const v = entry?.max_pages;
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
return DEFAULT_MAX_PAGES;
}
function toApiUrl(rawUrl) {
let u;
try { u = new URL(rawUrl); } catch { return null; }
if (u.protocol !== 'https:') return null;
if (!/^[a-z0-9-]+\.jibeapply\.com$/i.test(u.hostname)) return null;
if (!u.pathname.startsWith('/api/')) {
u.pathname = '/api' + (u.pathname.startsWith('/') ? u.pathname : '/' + u.pathname);
}
return u.href;
}
// Validate an explicit entry.api URL (any HTTPS hostname — used for iCIMS-hosted,
// branded JibeApply sites that share the same JSON schema).
function validateExplicitApi(apiUrl) {
let u;
try { u = new URL(apiUrl); } catch { return null; }
return u.protocol === 'https:' ? u.href : null;
}
export function parseJibeapplyResponse(json, entry) {
let origin = '';
try { origin = new URL(entry.careers_url || '').origin; } catch { /* ignore */ }
// careers_url isn't required to be a well-formed absolute URL for fetch()
// to succeed (an explicit entry.api bypasses it entirely — see toApiUrl
// callers below) — fall back to api's origin so job URLs stay absolute
// instead of silently degrading to a relative "/jobs/<slug>" path.
if (!origin) {
try { origin = new URL(entry.api || '').origin; } catch { /* ignore */ }
}
const items = Array.isArray(json?.jobs) ? json.jobs : [];
return items
.map(item => {
if (item == null) return null;
const d = item.data ?? item;
const title = String(d.title || '').trim();
const slug = d.slug || d.req_id;
if (!title || !slug) return null;
return {
title,
url: `${origin}/jobs/${encodeURIComponent(slug)}`,
company: String(d.hiring_organization || entry.name || '').trim(),
location: d.full_location || [d.city, d.country].filter(Boolean).join(', '),
};
})
.filter(Boolean);
}
/** @type {Provider} */
export default {
id: 'jibeapply',
detect(entry) {
const url = entry.careers_url;
if (typeof url !== 'string') return null;
const apiUrl = toApiUrl(url);
if (!apiUrl) return null;
return { url: apiUrl };
},
async fetch(entry, ctx) {
const url = entry.careers_url;
if (typeof url !== 'string' || !url) throw new Error('jibeapply: careers_url required');
// Prefer an explicit entry.api (allows iCIMS-hosted, branded JibeApply sites
// that share the same JSON schema but aren't on jibeapply.com).
const apiUrl = (typeof entry.api === 'string' && validateExplicitApi(entry.api))
|| toApiUrl(url);
if (!apiUrl) throw new Error(`jibeapply: cannot derive API URL for ${entry.name}`);
const first = await ctx.fetchJson(apiUrl, { redirect: 'error' });
const total = first.totalCount ?? 0;
// Use the actual number of items returned as page size — some implementations
// set `count` to the total rather than the per-page count.
const pageSize = first.jobs?.length || first.count || DEFAULT_PAGE_SIZE;
const allJobs = [...(first.jobs ?? [])];
if (total > pageSize && pageSize > 0) {
const maxPages = resolveMaxPages(entry);
const pages = Math.min(Math.ceil(total / pageSize), maxPages);
// Sequential, not concurrent (mirrors providers/4dayweek.mjs, thehub.mjs,
// arbeitnow.mjs, workday.mjs) — a single tenant's API has no reason to
// receive a burst of parallel requests, and a mid-run failure stops
// cleanly with whatever pages were already gathered instead of
// discarding them (Promise.all would fail the whole batch on one error).
for (let page = 2; page <= pages; page++) {
const u2 = new URL(apiUrl);
u2.searchParams.set('page', String(page));
let json;
try {
json = await ctx.fetchJson(u2.toString(), { redirect: 'error' });
} catch (err) {
console.error(`⚠️ jibeapply: ${entry.name} page ${page} fetch failed — ${err.message} (returning ${allJobs.length} jobs fetched so far)`);
break;
}
allJobs.push(...(json.jobs ?? []));
}
// The cap is silent by design (it's a safety net, not a working limit),
// but a tenant that actually exceeds it needs to be surfaced —
// otherwise the user has no way to notice postings are missing from
// their scan.
if (Math.ceil(total / pageSize) > maxPages) {
console.error(
`⚠️ jibeapply: ${entry.name} has more postings than max_pages allows ` +
`(fetched ${allJobs.length} of ${total}) — ` +
`set max_pages on this portal entry to raise the cap (current: ${maxPages})`,
);
}
}
return parseJibeapplyResponse({ jobs: allJobs }, entry);
},
};
+84
View File
@@ -0,0 +1,84 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Jobicy provider — board-wide remote-jobs aggregator feed
// (https://jobicy.com/api/v2/remote-jobs?count=50). Returns { jobs: [...] }.
//
// Wire in via a `job_boards:` entry with `provider: jobicy`.
const FEED_URL = 'https://jobicy.com/api/v2/remote-jobs?count=50';
/** @type {Provider} */
export default {
id: 'jobicy',
detect(entry) {
return entry?.provider === 'jobicy' ? { url: FEED_URL } : null;
},
/**
* Fetches and normalizes postings from the Jobicy public feed.
* @param {{ name?: string }} entry - The job_boards entry being processed.
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
* @returns {Promise<Array<{title: string, url: string, company: string, location: string, postedAt?: number}>>}
*/
async fetch(entry, ctx) {
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(FEED_URL, { redirect: 'error' });
if (!json || !Array.isArray(json.jobs)) {
throw new Error(`jobicy: unexpected API response — expected { jobs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
}
return parseJobicyResponse(json, entry.name || 'Jobicy');
},
};
/**
* Parse a Jobicy API response. Exported for unit tests.
*
* @param {any} json - Raw response payload.
* @param {string} defaultCompany - Fallback company name.
* @returns {Array<{title: string, url: string, company: string, location: string}>}
*/
export function parseJobicyResponse(json, defaultCompany = 'Jobicy') {
if (!json || !Array.isArray(json.jobs)) return [];
const toEpochMs = (value) => {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
};
return json.jobs
.map(j => {
if (!j || typeof j !== 'object') return null;
const title = typeof j.jobTitle === 'string' ? j.jobTitle.trim() : '';
if (!title) return null;
const rawUrl = typeof j.url === 'string' ? j.url.trim() : '';
let url = null;
try {
const parsed = new URL(rawUrl);
if (parsed.protocol === 'https:' && (parsed.hostname === 'jobicy.com' || parsed.hostname === 'www.jobicy.com')) {
url = parsed.href;
}
} catch {
// Invalid or malformed URL
}
if (!url) return null;
const company = typeof j.companyName === 'string' && j.companyName.trim() ? j.companyName.trim() : defaultCompany;
const location = typeof j.jobGeo === 'string' ? j.jobGeo.trim() : '';
const postedAt = toEpochMs(j.pubDate);
return {
title,
url,
company,
location,
postedAt,
};
})
.filter(j => j !== null);
}
+147
View File
@@ -0,0 +1,147 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Jobspresso provider - public WordPress jobs feed
// (https://jobspresso.co/?feed=job_feed). The feed is public, no-auth,
// and XML, so it is parsed in-process with the same tiny tag extractor
// approach as providers/personio.mjs rather than adding an XML dependency.
//
// Wire in via a `job_boards:` entry with `provider: jobspresso`.
const FEED_URL = 'https://jobspresso.co/?feed=job_feed';
const TRUSTED_HOST = 'jobspresso.co';
/** @param {string} url */
function assertJobspressoUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`jobspresso: invalid URL: ${url}`);
}
if (parsed.protocol !== "https:")
throw new Error(`jobspresso: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST) {
throw new Error(
`jobspresso: untrusted hostname "${parsed.hostname}" - must be ${TRUSTED_HOST}`,
);
}
return url;
}
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
/** @type {Provider} */
export default {
id: "jobspresso",
detect(entry) {
return entry?.provider === "jobspresso" ? { url: FEED_URL } : null;
},
async fetch(_entry, ctx) {
const feedUrl = assertJobspressoUrl(FEED_URL);
// redirect:'error' prevents SSRF via server-side redirects; combined with
// assertJobspressoUrl above it keeps the request pinned to jobspresso.co.
const text = await ctx.fetchText(feedUrl, { redirect: "error" });
return parseJobspressoFeed(text);
},
};
function fromCodePoint(cp) {
try {
return String.fromCodePoint(cp);
} catch {
return "";
}
}
// Decode the XML entities that appear in RSS text: numeric (&#38; / &#x27;)
// and the named five. Numeric forms are decoded first; &amp; is decoded LAST
// so a literal "&amp;lt;" yields "&lt;" rather than over-decoding to "<".
function decodeXmlEntities(s) {
return s
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, "&");
}
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
function extractText(inner) {
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
if (cdata) return cdata[1].trim();
return decodeXmlEntities(inner).trim();
}
// Extract the text of the first <tag>...</tag> in a block. Returns '' when absent.
function tagText(block, tag) {
const m = block.match(
new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, "i"),
);
return m ? extractText(m[1]) : "";
}
function cleanUrl(value) {
if (!value) return "";
const trimmed = value.trim();
try {
const parsed = new URL(trimmed);
const host = parsed.hostname.toLowerCase();
const trusted = host === TRUSTED_HOST || host.endsWith(`.${TRUSTED_HOST}`);
return parsed.protocol === "https:" && trusted ? parsed.href : "";
} catch {
return "";
}
}
/**
* Parse Jobspresso's public WordPress jobs feed.
*
* Shape:
* <rss><channel><item>...</item></channel></rss>
*
* Each item contains:
* - title
* - link
* - pubDate
* - job_listing:company
* - job_listing:location
*
* The RSS link is used as the dedup key.
* @param {string} xml - raw RSS feed body
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
*/
export function parseJobspressoFeed(xml) {
if (typeof xml !== "string") return [];
const jobs = [];
const blocks = xml.match(/<item\b[^>]*>[\s\S]*?<\/item>/gi) || [];
for (const item of blocks) {
const url = cleanUrl(tagText(item, "link"));
if (!url) continue;
const title = tagText(item, "title");
if (!title) continue;
const company = tagText(item, "job_listing:company") || "";
const location = tagText(item, "job_listing:location");
jobs.push({
title,
company,
location,
url,
postedAt: toEpochMs(tagText(item, "pubDate")),
});
}
return jobs;
}
+220
View File
@@ -0,0 +1,220 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Jobstreet / SEEK provider — hits the public SEEK v5 JobSearch REST API.
//
// Jobstreet (jobstreet.com, jobstreet.co.id, etc.) and SEEK (seek.com.au,
// seek.co.nz) share the same SEEK infrastructure. The old chalice-search
// v4 API (/api/chalice-search/v4/search) was deprecated; the v5 API at
// /api/jobsearch/v5/search is the current replacement.
//
// This provider is designed for explicit `provider: jobstreet` in portals.yml.
// Auto-detection from careers_url is not supported because Jobstreet is a
// job board aggregator, not a company ATS.
//
// Portal entry fields (all optional except `provider`):
// api — v5 search endpoint URL (default: https://id.jobstreet.com/api/jobsearch/v5/search)
// siteKey — SEEK site key for regional filtering (default: "ID-Main")
// searchKeywords — Search keywords, space-separated (default: "")
// searchLocation — Location filter (default: "")
// pageSize — Results per page (default: 30)
// maxPages — Maximum pages to fetch (default: 3)
//
// Site keys by market:
// ID-Main → id.jobstreet.com (Indonesia)
// SG-Main → sg.jobstreet.com (Singapore)
// MY-Main → my.jobstreet.com (Malaysia)
const DEFAULT_API = 'https://id.jobstreet.com/api/jobsearch/v5/search';
const DEFAULT_SITE_KEY = 'ID-Main';
const DEFAULT_PAGE_SIZE = 30;
const DEFAULT_MAX_PAGES = 3;
const ALLOWED_JOBSTREET_HOSTS = new Set([
'id.jobstreet.com',
'www.jobstreet.com',
'www.jobstreet.co.id',
'jobstreet.com',
'jobstreet.co.id',
'sg.jobstreet.com',
'my.jobstreet.com',
'www.seek.com.au',
'www.seek.co.nz',
]);
// v5 API paths (the client-side JS on jobstreet uses these relative paths
// resolved against the current origin). We keep the allowlist for SSRF
// protection on the base URL, then build the v5 search path from it.
const V5_SEARCH_PATH = '/api/jobsearch/v5/search';
/** @param {string} url */
function assertJobstreetUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`jobstreet: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`jobstreet: URL must use HTTPS: ${url}`);
if (!ALLOWED_JOBSTREET_HOSTS.has(parsed.hostname))
throw new Error(`jobstreet: untrusted hostname "${parsed.hostname}" — must be one of: ${[...ALLOWED_JOBSTREET_HOSTS].join(', ')}`);
return url;
}
/**
* Derive the origin from the API hostname.
* e.g. id.jobstreet.com → https://id.jobstreet.com
* @param {string} apiUrl
* @returns {string}
*/
function deriveOrigin(apiUrl) {
try {
const parsed = new URL(apiUrl);
return `${parsed.protocol}//${parsed.hostname}`;
} catch {
return 'https://id.jobstreet.com';
}
}
// NaN-safe Date.parse
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
/**
* Parse a single Jobstreet/SEEK v5 search API result into the canonical Job shape.
*
* The v5 search API returns objects shaped like:
* {
* id: "92996157",
* title: "Facility Engineer",
* advertiser: { id: "60960115", description: "PT YOFC International Indonesia" },
* companyName: "YOFC International",
* locations: [{ label: "Karawang, West Java", countryCode: "ID", ... }],
* listingDate: "2026-06-29T02:53:00Z",
* listingDateDisplay: "16h ago",
* roleId: "facilities-engineer",
* salaryLabel: "",
* teaser: "...",
* workTypes: ["Full time"],
* workArrangements: { data: [{ id: "1", label: { text: "On-site" } }] },
* ...
* }
*
* This parser is exported as a named export for unit tests.
*
* @param {any} item — raw v5 API result item
* @param {string} origin — scheme + hostname for building job detail URLs
* @param {string} fallbackCompany — company name fallback from the portal entry
* @returns {{title: string, url: string, company: string, location: string, postedAt: number|undefined}|null}
*/
export function parseJobstreetItem(item, origin, fallbackCompany) {
if (!item || typeof item !== 'object') return null;
const title = (item.title || '').trim();
if (!title) return null;
// Build job URL from the job ID
const jobId = (item.id || '').trim();
if (!jobId) return null;
const url = `${origin}/id/job/${jobId}`;
// Validate URL hostname belongs to allowed set
try {
const parsed = new URL(url);
if (!ALLOWED_JOBSTREET_HOSTS.has(parsed.hostname)) return null;
} catch {
return null;
}
// Prefer advertiser.description for the branded company name, fall back
// to companyName (which can be shorter/less specific), then entry name.
const company = (item.advertiser?.description || item.companyName || fallbackCompany || '').trim();
const location = (item.locations?.[0]?.label || '').trim();
const postedAt = toEpochMs(item.listingDate);
return { title, url, company, location, ...(postedAt != null ? { postedAt } : {}) };
}
/**
* Build the v5 search URL with query parameters.
* @param {string} origin — scheme + hostname
* @param {object} params
* @returns {string}
*/
function buildSearchUrl(origin, params) {
const url = new URL(V5_SEARCH_PATH, origin);
const { siteKey, keywords, location, pageSize, page } = params;
if (siteKey) url.searchParams.set('siteKey', siteKey);
if (keywords) url.searchParams.set('keywords', keywords);
if (location) url.searchParams.set('where', location);
url.searchParams.set('pageSize', String(pageSize || DEFAULT_PAGE_SIZE));
url.searchParams.set('page', String(page || 1));
return url.href;
}
/** @type {Provider} */
export default {
id: 'jobstreet',
detect(_entry) {
// Jobstreet is a job board aggregator, not a company ATS.
// Auto-detection from careers_url is intentionally not supported —
// use `provider: jobstreet` explicitly in portals.yml.
return null;
},
async fetch(entry, ctx) {
const apiUrl = entry.api || DEFAULT_API;
assertJobstreetUrl(apiUrl);
const origin = deriveOrigin(apiUrl);
const siteKey = entry.siteKey || DEFAULT_SITE_KEY;
const keywords = entry.searchKeywords || '';
const searchLocation = entry.searchLocation || '';
const pageSize = Number(entry.pageSize) || DEFAULT_PAGE_SIZE;
const maxPages = Number(entry.maxPages) || DEFAULT_MAX_PAGES;
const fallbackCompany = entry.name || '';
const allJobs = [];
for (let page = 1; page <= maxPages; page++) {
const searchUrl = buildSearchUrl(origin, {
siteKey,
keywords,
location: searchLocation,
pageSize,
page,
});
let json;
try {
json = /** @type {any} */ (await ctx.fetchJson(searchUrl, { redirect: 'error' }));
} catch (err) {
// If page 1 fails, surface the error. Later pages failing is non-fatal
// — we return whatever we've collected so far.
if (page === 1) throw err;
console.error(`jobstreet: page ${page} fetch failed — ${err.message}`);
break;
}
const data = Array.isArray(json?.data) ? json.data : [];
if (data.length === 0) break;
for (const item of data) {
const job = parseJobstreetItem(item, origin, fallbackCompany);
if (job) allJobs.push(job);
}
// Stop if we got fewer results than pageSize (last page)
if (data.length < pageSize) break;
// Respect rate limits — small delay between pages
await new Promise(resolve => setTimeout(resolve, 200));
}
return allJobs;
},
};
+128
View File
@@ -0,0 +1,128 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// JustJoin.it provider — hits the current candidate offers API.
// Browser URLs under https://justjoin.it/job-offers/... are accepted for
// detection, but fetches use https://justjoin.it/api/candidate-api/offers.
const ALLOWED_HOSTS = new Set(['justjoin.it']);
const API_BASE = 'https://justjoin.it/api/candidate-api/offers';
const JOB_BASE = 'https://justjoin.it/job-offer/';
const PAGE_SIZE = 100;
const MAX_PAGES = 50;
function assertJustJoinUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`justjoin: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`justjoin: URL must use HTTPS: ${url}`);
if (!ALLOWED_HOSTS.has(parsed.hostname)) {
throw new Error(`justjoin: untrusted hostname "${parsed.hostname}" — must be justjoin.it`);
}
if (!parsed.pathname.startsWith('/job-offers') && parsed.pathname !== '/api/candidate-api/offers') {
throw new Error(`justjoin: URL path must be /job-offers or /api/candidate-api/offers: ${url}`);
}
return parsed;
}
function detectUrl(entry) {
const url = entry.api || entry.careers_url || '';
if (typeof url !== 'string' || !url.trim()) return null;
try {
const parsed = assertJustJoinUrl(url);
return { url: parsed.href };
} catch {
return null;
}
}
function normalizeLocation(offer) {
const parts = [];
const workplace = String(offer?.workplaceType || '').trim();
if (workplace) parts.push(workplace);
if (Array.isArray(offer?.locations)) {
for (const location of offer.locations) {
const city = String(location?.city || '').trim();
if (city) parts.push(city);
}
}
const city = String(offer?.city || '').trim();
if (city) parts.push(city);
return [...new Set(parts.filter(Boolean))].join(', ');
}
function postedAtMillis(value) {
const time = Date.parse(String(value || ''));
return Number.isFinite(time) ? time : undefined;
}
function buildApiUrl(entry, from) {
const apiUrl = entry.api || API_BASE;
const parsed = assertJustJoinUrl(apiUrl);
if (parsed.pathname !== '/api/candidate-api/offers') {
parsed.pathname = '/api/candidate-api/offers';
parsed.search = '';
}
parsed.searchParams.set('from', String(from));
parsed.searchParams.set('itemsCount', String(Number(entry.page_size || PAGE_SIZE)));
parsed.searchParams.set('cityRadius', String(Number(entry.city_radius || 30)));
parsed.searchParams.set('currency', String(entry.currency || 'pln').toLowerCase());
parsed.searchParams.set('orderBy', 'descending');
parsed.searchParams.set('sortBy', 'publishedAt');
parsed.searchParams.set('keywordType', 'any');
parsed.searchParams.set('isPromoted', 'true');
return parsed.href;
}
export function parseJustJoinResponse(json) {
if (!json || !Array.isArray(json.data)) {
throw new Error(`justjoin: unexpected API response — expected { data: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
}
return json.data
.filter(offer => offer && typeof offer === 'object')
.map(offer => {
const slug = String(offer.slug || '').trim();
const title = String(offer.title || '').trim();
if (!slug || !title) return null;
return {
title,
url: `${JOB_BASE}${slug}`,
company: String(offer.companyName || '').trim(),
location: normalizeLocation(offer),
postedAt: postedAtMillis(offer.publishedAt),
};
})
.filter(Boolean);
}
/** @type {Provider} */
export default {
id: 'justjoin',
detect(entry) {
return detectUrl(entry);
},
async fetch(entry, ctx) {
if (!detectUrl(entry)) throw new Error('justjoin: careers_url or api must be a trusted justjoin.it URL');
const jobs = [];
let from = 0;
const maxPages = Number(entry.max_pages || MAX_PAGES);
for (let page = 0; page < maxPages; page++) {
const url = buildApiUrl(entry, from);
const json = await ctx.fetchJson(url, { redirect: 'error' });
const parsed = parseJustJoinResponse(json);
jobs.push(...parsed);
const next = json?.meta?.next?.cursor;
if (next == null || parsed.length === 0) break;
from = Number(next);
if (!Number.isFinite(from)) break;
}
return jobs;
},
};
+132
View File
@@ -0,0 +1,132 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Landing.jobs provider — board-wide aggregator feed (tech, Europe-focused):
// https://landing.jobs/api/v1/jobs
// Response shape: a JSON ARRAY of
// { id, title, url, locations: [{ city, country_code }], remote, published_at,
// created_at, type, tags, gross_salary_low, gross_salary_high, ... }
//
// NOTE: the v1 feed carries no company-name field — the employer slug only
// appears in the posting URL path (`https://landing.jobs/at/<slug>/<job>`), so
// `company` is derived best-effort from that slug (humanized) and falls back to
// the portal entry name. The whole active set is returned in one call.
//
// Wire in via a `job_boards:` entry with `provider: landingjobs`.
const FEED_URL = 'https://landing.jobs/api/v1/jobs';
const TRUSTED_HOST = 'landing.jobs';
/** @param {string} url */
function assertLandingUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`landingjobs: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`landingjobs: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST) {
throw new Error(`landingjobs: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
}
return url;
}
// NaN-safe Date.parse.
function toEpochMs(value) {
if (typeof value !== 'string' || !value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
/**
* Derive a best-effort company name from a Landing.jobs posting URL.
* Posting URLs are `https://landing.jobs/at/<slug>/<job>`; the `<slug>` is
* humanized (hyphens/underscores → spaces, title-cased). Returns '' when the
* URL is not the expected `/at/<slug>/…` shape. Exported for unit tests.
* @param {string} url
*/
export function companyFromUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
return '';
}
const segs = parsed.pathname.split('/').filter(Boolean);
if (segs[0] !== 'at' || !segs[1]) return '';
return segs[1]
.replace(/[-_]+/g, ' ')
.trim()
.replace(/\b\w/g, c => c.toUpperCase());
}
/**
* Normalize a single Landing.jobs job. Exported for unit tests.
*
* Field mapping → the normalized Job shape:
* - title: `title`, trimmed (postings without one are dropped).
* - url: `url` — host-locked to `landing.jobs` (an off-host or non-https
* URL drops the posting). It is the dedup key and is display-only.
* - company: derived from the URL slug (see companyFromUrl), falling back to
* the portal entry name, then "Landing.jobs".
* - location: first `locations[]` entry as "city, country_code"; "Remote" is
* appended when `remote` is true.
* - postedAt: `published_at` (else `created_at`) ISO date → epoch ms.
*
* @param {any} j
* @param {string} [fallbackCompany]
* @returns {{ title: string, url: string, company: string, location: string, postedAt?: number } | null}
*/
export function normalizeLandingJob(j, fallbackCompany) {
if (!j || typeof j !== 'object') return null;
const title = typeof j.title === 'string' ? j.title.trim() : '';
if (!title) return null;
let url = '';
const rawUrl = typeof j.url === 'string' ? j.url.trim() : '';
if (rawUrl) {
try {
const parsed = new URL(rawUrl);
if (parsed.protocol === 'https:' && parsed.hostname === TRUSTED_HOST) url = parsed.href;
} catch {
// malformed URL → leave url = '' → dropped below
}
}
if (!url) return null;
const company =
companyFromUrl(url) ||
(typeof fallbackCompany === 'string' && fallbackCompany.trim() ? fallbackCompany.trim() : 'Landing.jobs');
const first = Array.isArray(j.locations) && j.locations[0] && typeof j.locations[0] === 'object' ? j.locations[0] : {};
const city = typeof first.city === 'string' ? first.city.trim() : '';
const country = typeof first.country_code === 'string' ? first.country_code.trim() : '';
const base = [city, country].filter(Boolean).join(', ');
const location = [base, j.remote === true ? 'Remote' : ''].filter(Boolean).join(', ');
/** @type {{ title: string, url: string, company: string, location: string, postedAt?: number }} */
const job = { title, url, company, location };
const postedAt = toEpochMs(j.published_at) ?? toEpochMs(j.created_at);
if (postedAt !== undefined) job.postedAt = postedAt;
return job;
}
/** @type {Provider} */
export default {
id: 'landingjobs',
async fetch(entry, ctx) {
assertLandingUrl(FEED_URL);
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(FEED_URL, { redirect: 'error' });
if (!Array.isArray(json)) {
throw new Error(
`landingjobs: unexpected API response — expected a JSON array, got ${json === null ? 'null' : typeof json}`,
);
}
const fallbackCompany = entry?.name;
return json.map(j => normalizeLandingJob(j, fallbackCompany)).filter(Boolean);
},
};
+39
View File
@@ -0,0 +1,39 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Lever provider — hits the public postings endpoint.
// Auto-detects from careers_url pattern `https://jobs.lever.co/<slug>`.
function resolveApiUrl(entry) {
const url = entry.careers_url || '';
const match = url.match(/jobs\.lever\.co\/([^/?#]+)/);
if (!match) return null;
return `https://api.lever.co/v0/postings/${match[1]}`;
}
/** @type {Provider} */
export default {
id: 'lever',
detect(entry) {
const apiUrl = resolveApiUrl(entry);
return apiUrl ? { url: apiUrl } : null;
},
async fetch(entry, ctx) {
const apiUrl = resolveApiUrl(entry);
if (!apiUrl) throw new Error(`lever: cannot derive API URL for ${entry.name}`);
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
if (!Array.isArray(json)) return [];
return json.map(j => ({
title: j.text || '',
url: j.hostedUrl || '',
company: entry.name,
location: j.categories?.location || '',
// Lever's v0 postings list ships the full description for free (same
// payload, no per-job request) — enables scan.mjs content_filter.
description: typeof j.descriptionPlain === 'string' ? j.descriptionPlain : '',
postedAt: typeof j.createdAt === 'number' ? j.createdAt : undefined,
}));
},
};
+211
View File
@@ -0,0 +1,211 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
import { execFile } from 'child_process';
import { realpathSync } from 'fs';
import { resolve, sep } from 'path';
import { fileURLToPath } from 'url';
import { promisify } from 'util';
const execFileAsync = promisify(execFile);
const LOCAL_PARSER_TIMEOUT_MS = 20_000;
const LOCAL_PARSER_MAX_BUFFER_BYTES = 2_000_000;
// `parser.command` / `parser.script` come from portals.yml, which on a shared or
// template config is not fully trusted. The command must be a known interpreter
// or a file inside this project — never an arbitrary binary like `rm` or `curl`.
const PROJECT_ROOT = realpathSync(resolve(fileURLToPath(new URL('..', import.meta.url))));
const ALLOWED_INTERPRETERS = new Set(['python3', 'python', 'node', 'deno', 'bun', 'sh', 'bash']);
// `{careers_url}` and `{company}` are interpolated into the parser's argv. Validate
// them so an interpolated value can never be read as a CLI flag (argument injection).
function safeCareersUrl(value) {
if (!value) return '';
let url;
try {
url = new URL(String(value));
} catch {
throw new Error(`local-parser: careers_url is not a valid URL: ${value}`);
}
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
throw new Error(`local-parser: careers_url must be http(s): ${value}`);
}
return url.href;
}
function safeCompany(value) {
if (!value) return '';
const name = String(value).trim();
// execFile passes args verbatim (no shell), so the only injection risk is a
// value that begins like a CLI flag.
if (name.startsWith('-')) {
throw new Error(`local-parser: company name cannot start with '-': ${value}`);
}
return name;
}
// Only validate a placeholder's value when the arg actually uses it — a fixed
// `parser.script` must not be rejected because some unrelated `{company}` value
// has punctuation it never sees.
function expandParserArg(value, entry) {
let out = String(value);
if (out.includes('{careers_url}')) out = out.replaceAll('{careers_url}', safeCareersUrl(entry.careers_url));
if (out.includes('{company}')) out = out.replaceAll('{company}', safeCompany(entry.name));
return out;
}
function getParserScriptPath(entry) {
const parser = entry.parser || {};
if (parser.script) return expandParserArg(parser.script, entry);
const args = Array.isArray(parser.args) ? parser.args : [];
const scriptArg = args.find(arg => {
const value = String(arg);
return !value.startsWith('-') && /\.(py|mjs|js|sh)$/.test(value);
});
return scriptArg ? expandParserArg(scriptArg, entry) : null;
}
function buildParserArgs(entry) {
const parser = entry.parser || {};
const args = [];
if (parser.script) args.push(parser.script);
if (Array.isArray(parser.args)) args.push(...parser.args);
return args.map(arg => expandParserArg(arg, entry));
}
// Resolve a configured path and confirm it stays inside the project tree.
function resolveInsideRoot(rawPath) {
const resolved = realpathSync(resolve(PROJECT_ROOT, String(rawPath)));
if (resolved !== PROJECT_ROOT && !resolved.startsWith(PROJECT_ROOT + sep)) {
throw new Error(`local-parser: path escapes the project root: ${rawPath}`);
}
return resolved;
}
// The command is either a whitelisted interpreter (resolved via PATH) or a script
// that lives inside the repo. Anything else is rejected.
function resolveCommand(command) {
const value = String(command || '');
if (!value) throw new Error('local-parser: parser.command is required');
if (!value.includes('/') && ALLOWED_INTERPRETERS.has(value)) return value;
return resolveInsideRoot(value);
}
// Validate the whole invocation and return what to spawn. Throws on anything unsafe.
function resolveInvocation(entry) {
const rawCommand = String(entry.parser?.command || '');
const command = resolveCommand(rawCommand);
const args = buildParserArgs(entry);
const scriptPath = getParserScriptPath(entry);
const usesInterpreter = !rawCommand.includes('/') && ALLOWED_INTERPRETERS.has(rawCommand);
if (usesInterpreter) {
// A whitelisted interpreter must run an in-repo script as its FIRST argument.
// Anything before the script is an interpreter option (node --eval / --require,
// python -c, …) that could execute arbitrary code, so require the script to lead.
if (!scriptPath) throw new Error('local-parser: interpreter command requires an in-repo parser script');
resolveInsideRoot(scriptPath);
if (args[0] !== scriptPath) {
throw new Error('local-parser: the parser script must be the interpreter\'s first argument');
}
} else if (scriptPath) {
// command is an in-repo file; keep any detected script path inside the repo too.
resolveInsideRoot(scriptPath);
}
return { command, args };
}
function normalizeJobUrl(rawUrl, baseUrl) {
if (!rawUrl) return '';
try {
return new URL(String(rawUrl).trim(), baseUrl || undefined).href;
} catch {
return '';
}
}
function normalizeLocation(value) {
if (!value) return '';
if (Array.isArray(value)) return value.map(normalizeLocation).filter(Boolean).join(', ');
if (typeof value === 'object') return value.name || value.text || '';
return String(value).trim();
}
function normalizeParserJob(job, entry) {
if (!job || typeof job !== 'object') return null;
const title = String(job.title || job.name || '').trim();
const url = normalizeJobUrl(
job.url || job.jobUrl || job.job_url || job.applyUrl || job.apply_url,
entry.careers_url,
);
if (!title || !url) return null;
return {
title,
url,
company: String(job.company || entry.name || '').trim(),
location: normalizeLocation(job.location || job.locations),
};
}
async function runLocalParser(entry) {
const parser = entry.parser || {};
const { command, args } = resolveInvocation(entry);
const timeout = Number(parser.timeout_ms || LOCAL_PARSER_TIMEOUT_MS);
const maxBuffer = Number(parser.max_buffer_bytes || LOCAL_PARSER_MAX_BUFFER_BYTES);
// cwd is pinned to the project root so a relative script arg resolves to the
// same file resolveInvocation() validated, regardless of the caller's cwd.
const { stdout } = await execFileAsync(command, args, {
cwd: PROJECT_ROOT,
timeout,
maxBuffer,
windowsHide: true,
});
let payload;
try {
payload = JSON.parse(stdout);
} catch {
throw new Error('local parser returned invalid JSON');
}
const rawJobs = Array.isArray(payload) ? payload : payload.jobs || payload.results;
if (!Array.isArray(rawJobs)) {
throw new Error('local parser JSON must be an array or contain jobs[]/results[]');
}
return rawJobs
.map(job => normalizeParserJob(job, entry))
.filter(Boolean);
}
/** @type {Provider} */
export default {
id: 'local-parser',
detect(entry) {
if (!entry.parser?.command) return null;
// An invocation we can't safely resolve (unknown command, out-of-repo or
// missing script, inline-code flags) is not runnable — skip it.
try {
resolveInvocation(entry);
} catch {
return null;
}
return { url: entry.careers_url || 'local-parser' };
},
async fetch(entry) {
return runLocalParser(entry);
},
};
+157
View File
@@ -0,0 +1,157 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// NoDesk provider - board-wide RSS feed
// (https://nodesk.co/remote-jobs/index.xml). The feed is public, no-auth,
// and XML, so it is parsed in-process with the same tiny tag extractor
// approach as providers/personio.mjs rather than adding an XML dependency.
//
// Wire in via a `job_boards:` entry with `provider: nodesk`.
const FEED_URL = 'https://nodesk.co/remote-jobs/index.xml';
const TRUSTED_HOST = 'nodesk.co';
/** @param {string} url */
function assertNodeskUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`nodesk: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`nodesk: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST) {
throw new Error(`nodesk: untrusted hostname "${parsed.hostname}" - must be ${TRUSTED_HOST}`);
}
return url;
}
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
function fallbackCompany(entry) {
return typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : 'NoDesk';
}
/** @type {Provider} */
export default {
id: 'nodesk',
detect(entry) {
return entry?.provider === 'nodesk' ? { url: FEED_URL } : null;
},
async fetch(entry, ctx) {
const feedUrl = assertNodeskUrl(FEED_URL);
// redirect:'error' prevents SSRF via server-side redirects; combined with
// assertNodeskUrl above it keeps the request pinned to nodesk.co.
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
return parseNodeskFeed(text, fallbackCompany(entry));
},
};
function fromCodePoint(cp) {
try {
return String.fromCodePoint(cp);
} catch {
return '';
}
}
// Decode the XML entities that appear in RSS text: numeric (&#38; / &#x27;)
// and the named five. Numeric forms are decoded first; &amp; is decoded LAST
// so a literal "&amp;lt;" yields "&lt;" rather than over-decoding to "<".
function decodeXmlEntities(s) {
return s
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, '&');
}
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
function extractText(inner) {
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
if (cdata) return cdata[1].trim();
return decodeXmlEntities(inner).trim();
}
// Extract the text of the first <tag>...</tag> in a block. Returns '' when absent.
function tagText(block, tag) {
const m = block.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, 'i'));
return m ? extractText(m[1]) : '';
}
// Keep only absolute HTTPS links hosted on the trusted board domain.
function cleanUrl(value) {
if (!value) return '';
const trimmed = value.trim();
try {
const parsed = new URL(trimmed);
const host = parsed.hostname.toLowerCase();
const trusted = host === TRUSTED_HOST || host.endsWith(`.${TRUSTED_HOST}`);
return parsed.protocol === 'https:' && trusted ? parsed.href : '';
} catch {
return '';
}
}
// NoDesk encodes the company in the RSS title as "Role at Company".
function splitTitle(rawTitle, defaultCompany) {
const text = rawTitle.trim();
const lower = text.toLowerCase();
const idx = lower.lastIndexOf(' at ');
if (idx > 0) {
const title = text.slice(0, idx).trim();
const company = text.slice(idx + 4).trim();
if (title && company) return { title, company };
}
return { title: text, company: defaultCompany };
}
/**
* Parse NoDesk's public RSS jobs feed. Exported for unit tests.
*
* Shape: `<rss><channel><item>...</item>...</channel></rss>`. Each item
* exposes `<title>`, `<link>`, and `<pubDate>`. NoDesk currently encodes the
* company inside the title as `Role at Company`; there is no dedicated
* location tag in the feed, so location stays empty unless the feed evolves.
*
* @param {string} xml - raw RSS feed body
* @param {string} [defaultCompany] - fallback company for unsplittable titles
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
*/
export function parseNodeskFeed(xml, defaultCompany = 'NoDesk') {
if (typeof xml !== 'string') return [];
const fallback = typeof defaultCompany === 'string' && defaultCompany.trim() ? defaultCompany.trim() : 'NoDesk';
const jobs = [];
const blocks = xml.match(/<item\b[^>]*>[\s\S]*?<\/item>/gi) || [];
for (const item of blocks) {
const url = cleanUrl(tagText(item, 'link'));
if (!url) continue;
const rawTitle = tagText(item, 'title');
if (!rawTitle) continue;
const { title, company } = splitTitle(rawTitle, fallback);
const postedAt = toEpochMs(tagText(item, 'pubDate'));
const job = {
title,
company,
location: '',
url,
};
if (postedAt !== undefined) job.postedAt = postedAt;
jobs.push(job);
}
return jobs;
}
+167
View File
@@ -0,0 +1,167 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// NoFluffJobs provider — hits the public search posting API.
// It intentionally returns only the core scanner job fields; richer skill and
// salary metadata can be added later if the provider contract is expanded.
const ALLOWED_HOSTS = new Set(['nofluffjobs.com']);
const API_URL = 'https://nofluffjobs.com/api/search/posting';
const JOB_BASE = 'https://nofluffjobs.com/pl/job/';
const PAGE_SIZE = 20;
const MAX_PAGES = 5;
function assertNoFluffUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`nofluffjobs: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`nofluffjobs: URL must use HTTPS: ${url}`);
if (!ALLOWED_HOSTS.has(parsed.hostname)) {
throw new Error(`nofluffjobs: untrusted hostname "${parsed.hostname}" — must be nofluffjobs.com`);
}
return parsed;
}
function detectUrl(entry) {
const url = entry.api || entry.careers_url || '';
if (typeof url !== 'string' || !url.trim()) return null;
try {
return { url: assertNoFluffUrl(url).href };
} catch {
return null;
}
}
function normalizeLocation(posting) {
const parts = [];
if (posting?.fullyRemote || posting?.location?.fullyRemote) parts.push('Remote');
if (Array.isArray(posting?.location?.places)) {
for (const place of posting.location.places) {
const city = String(place?.city || '').trim();
const province = String(place?.province || '').trim();
const country = String(place?.country?.name || '').trim();
if (city) parts.push(city);
else if (province) parts.push(province);
else if (country) parts.push(country);
}
}
return [...new Set(parts.filter(Boolean))].join(', ');
}
function postedAtMillis(value) {
const number = Number(value);
return Number.isFinite(number) ? number : undefined;
}
function buildRequest(entry, pageTo) {
const apiUrl = assertNoFluffUrl(entry.api || API_URL);
apiUrl.pathname = '/api/search/posting';
apiUrl.search = '';
apiUrl.searchParams.set('sort', 'newest');
apiUrl.searchParams.set('withSalaryMatch', 'true');
apiUrl.searchParams.set('pageTo', String(pageTo));
apiUrl.searchParams.set('pageSize', String(Number(entry.page_size || PAGE_SIZE)));
apiUrl.searchParams.set('salaryCurrency', String(entry.currency || 'PLN').toUpperCase());
apiUrl.searchParams.set('salaryPeriod', String(entry.salary_period || 'month'));
apiUrl.searchParams.set('region', String(entry.region || 'pl'));
apiUrl.searchParams.set('language', String(entry.language || 'pl-PL'));
const rawSearch = String(entry.search || entry.raw_search || '').trim();
const body = rawSearch
? {
criteria: '',
url: { searchParam: rawSearch },
rawSearch,
pageSize: Number(entry.page_size || PAGE_SIZE),
withSalaryMatch: true,
}
: {
criteriaSearch: {
country: [],
withSalaryMatch: [],
city: [],
more: [],
employment: [],
requirement: [],
salary: [],
jobPosition: [],
applicationStatus: [],
province: [],
company: [],
id: [],
category: [],
keyword: [],
jobLanguage: [],
seniority: [],
},
pageSize: Number(entry.page_size || PAGE_SIZE),
withSalaryMatch: true,
};
return { url: apiUrl.href, body };
}
export function parseNoFluffJobsResponse(json) {
if (!json || !Array.isArray(json.postings)) {
throw new Error(`nofluffjobs: unexpected API response — expected { postings: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
}
return json.postings
.filter(posting => posting && typeof posting === 'object')
.map(posting => {
const title = String(posting.title || '').trim();
const company = String(posting.name || '').trim();
const slug = String(posting.url || posting.id || '').trim();
if (!title || !slug) return null;
return {
title,
url: `${JOB_BASE}${slug}`,
company,
location: normalizeLocation(posting),
postedAt: postedAtMillis(posting.posted),
};
})
.filter(Boolean);
}
/** @type {Provider} */
export default {
id: 'nofluffjobs',
detect(entry) {
return detectUrl(entry);
},
async fetch(entry, ctx) {
if (!detectUrl(entry)) throw new Error('nofluffjobs: careers_url or api must be a trusted nofluffjobs.com URL');
const maxPages = Number(entry.max_pages || MAX_PAGES);
const jobs = [];
const seen = new Set();
for (let pageTo = 1; pageTo <= maxPages; pageTo++) {
const { url, body } = buildRequest(entry, pageTo);
const json = await ctx.fetchJson(url, {
method: 'POST',
body: JSON.stringify(body),
headers: {
accept: 'application/json, text/plain, */*',
'content-type': 'application/infiniteSearch+json',
},
redirect: 'error',
});
for (const job of parseNoFluffJobsResponse(json)) {
if (seen.has(job.url)) continue;
seen.add(job.url);
jobs.push(job);
}
const totalPages = Number(json?.totalPages || 0);
if (totalPages && pageTo >= totalPages) break;
if (json.postings.length === 0) break;
}
return jobs;
},
};
+169
View File
@@ -0,0 +1,169 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Personio provider — hits the public, no-auth XML jobs feed at
// `https://<slug>.jobs.personio.de/xml` (common across DACH/EU companies).
// Auto-detects from a `<slug>.jobs.personio.(de|com)` careers host like
// workable/recruitee. Per-tenant subdomains are the variable part, so the
// SSRF defence is an anchored host regex rather than a static allowlist.
//
// The feed is a flat, well-defined XML document, so it is parsed in-process
// with a tiny tag extractor (no new dependency — the repo ships none for XML).
const PERSONIO_HOST_RE = /^[a-z0-9][a-z0-9-]*\.jobs\.personio\.(de|com)$/;
/** @param {string} url */
function assertPersonioUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`personio: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`personio: URL must use HTTPS: ${url}`);
if (!PERSONIO_HOST_RE.test(parsed.hostname))
throw new Error(`personio: untrusted hostname "${parsed.hostname}" — must match <slug>.jobs.personio.(de|com)`);
return url;
}
/**
* Resolve the tenant host (e.g. `acme.jobs.personio.de`) from a careers_url.
* Returns null for non-Personio or malformed URLs.
* @param {import('./_types.js').PortalEntry} entry
*/
function resolveHost(entry) {
const raw = typeof entry.careers_url === 'string' ? entry.careers_url : '';
if (!raw) return null;
let parsed;
try {
parsed = new URL(raw);
} catch {
return null;
}
if (parsed.protocol !== 'https:') return null;
if (!PERSONIO_HOST_RE.test(parsed.hostname)) return null;
return parsed.hostname;
}
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
/** @type {Provider} */
export default {
id: 'personio',
detect(entry) {
const host = resolveHost(entry);
return host ? { url: `https://${host}/xml` } : null;
},
async fetch(entry, ctx) {
const host = resolveHost(entry);
if (!host) throw new Error(`personio: cannot derive feed URL for ${entry.name}`);
const feedUrl = `https://${host}/xml`;
assertPersonioUrl(feedUrl);
// redirect:'error' prevents SSRF via server-side redirects; combined with
// assertPersonioUrl above it guarantees the final hostname stays in-domain.
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
return parsePersonioXml(text, entry.name, host);
},
};
function fromCodePoint(cp) {
try {
return String.fromCodePoint(cp);
} catch {
return '';
}
}
// Decode the XML entities that appear in Personio job text: numeric (&#38; /
// &#x27;) and the named five. Numeric forms are decoded first; &amp; is decoded
// LAST so a literal "&amp;lt;" yields "&lt;" rather than over-decoding to "<".
function decodeXmlEntities(s) {
return s
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, '&');
}
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
function extractText(inner) {
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
if (cdata) return cdata[1].trim();
return decodeXmlEntities(inner).trim();
}
// Extract the text of the first <tag>…</tag> in a block. Returns '' when absent.
function tagText(block, tag) {
const m = block.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`));
return m ? extractText(m[1]) : '';
}
/**
* Parse Personio's public XML jobs feed. Exported for unit tests.
*
* Shape: `<workzag-jobs><position>…</position>…</workzag-jobs>`, each position
* carrying `<id>`, `<name>`, `<office>` (+ optional `<additionalOffices><office>`),
* and `<createdAt>` (ISO 8601). The feed has NO per-job URL, so it is built from
* the already-validated tenant host: `https://<host>/job/<id>`.
*
* - title: `<name>` (required — positions without one are dropped).
* - url: `https://<host>/job/<id>` — only when `<id>` is a plain integer, so a
* malformed id can never inject into the URL. url is the dedup key; a position
* without a usable id is dropped.
* - location: every `<office>` in the block (primary + additionalOffices),
* de-duplicated, joined with ", ".
* - postedAt: `<createdAt>` → epoch ms (omitted when unparseable/absent).
*
* @param {string} xml — raw XML feed body
* @param {string} companyName — value written into job.company
* @param {string} host — validated tenant host, e.g. `acme.jobs.personio.de`
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
*/
export function parsePersonioXml(xml, companyName, host) {
if (typeof xml !== 'string') return [];
const jobs = [];
// Strip every <jobDescriptions> subtree from the WHOLE feed before splitting
// into <position> blocks: descriptions are free-text HTML that can carry a
// literal "</position>" which would otherwise truncate the non-greedy block
// match. It also drops the per-section <name>/<value> pairs whose nested
// <name> would race the position's own <name> (same for any other scalar tag).
const stripped = xml.replace(/<jobDescriptions\b[^>]*>[\s\S]*?<\/jobDescriptions>/gi, '');
const blocks = stripped.match(/<position\b[^>]*>[\s\S]*?<\/position>/g) || [];
for (const scalar of blocks) {
const title = tagText(scalar, 'name');
if (!title) continue;
const id = tagText(scalar, 'id');
if (!/^\d+$/.test(id)) continue; // need a clean numeric id to build the url
// Collect every <office> (primary + additionalOffices), de-dupe, join.
const offices = [];
const seen = new Set();
for (const om of scalar.matchAll(/<office\b[^>]*>([\s\S]*?)<\/office>/g)) {
const name = extractText(om[1]);
if (name && !seen.has(name)) {
seen.add(name);
offices.push(name);
}
}
jobs.push({
title,
url: `https://${host}/job/${id}`,
location: offices.join(', '),
company: companyName,
postedAt: toEpochMs(tagText(scalar, 'createdAt')),
});
}
return jobs;
}
+128
View File
@@ -0,0 +1,128 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Pinpoint provider — hits the public per-tenant postings.json feed.
// Auto-detects from careers_url pattern `https://<slug>.pinpointhq.com`.
//
// Pinpoint's postings.json is public, zero-auth and returns every active
// posting for the tenant in a single call:
// https://<slug>.pinpointhq.com/postings.json
// Response shape: { data: [ { title, url, path, location: { name, city,
// province, ... }, job: { department, division, ... }, compensation, ... } ] }
//
// Per-tenant subdomains are the variable part — SSRF defence uses a regex
// match on `<safe-slug>.pinpointhq.com` rather than a static allowlist, the
// same approach as the recruitee provider.
// The tenant label must be a valid DNS label: it may contain hyphens but must
// not start or end with one (so `acme-.pinpointhq.com` is rejected). The
// optional trailing group keeps single-character labels (e.g. `a.pinpointhq.com`)
// valid. detect() and fetch() both route through this constant via
// resolveApiUrl()/assertPinpointUrl(), so the stricter check applies everywhere.
const PINPOINT_HOST_RE = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.pinpointhq\.com$/;
/** @param {string} url */
function assertPinpointUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`pinpoint: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`pinpoint: URL must use HTTPS: ${url}`);
if (!PINPOINT_HOST_RE.test(parsed.hostname)) {
throw new Error(`pinpoint: untrusted hostname "${parsed.hostname}" — must match <slug>.pinpointhq.com`);
}
return url;
}
function resolveApiUrl(entry) {
const raw = typeof entry.careers_url === 'string' ? entry.careers_url : '';
if (!raw) return null;
let parsed;
try {
parsed = new URL(raw);
} catch {
return null;
}
if (parsed.protocol !== 'https:') return null;
if (!PINPOINT_HOST_RE.test(parsed.hostname)) return null;
return `https://${parsed.hostname}/postings.json`;
}
/** @type {Provider} */
export default {
id: 'pinpoint',
detect(entry) {
const apiUrl = resolveApiUrl(entry);
return apiUrl ? { url: apiUrl } : null;
},
async fetch(entry, ctx) {
const apiUrl = resolveApiUrl(entry);
if (!apiUrl) throw new Error(`pinpoint: cannot derive API URL for ${entry.name}`);
assertPinpointUrl(apiUrl);
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
return parsePinpointResponse(json, entry.name);
},
};
/**
* Parse a Pinpoint /postings.json response. Exported for unit tests.
*
* Pinpoint returns:
* { data: [{ title, url, path, location: { name, city, province, ... }, ... }] }
*
* Field mapping → the normalized Job shape:
* - title: `title`, trimmed.
* - url: `url` — an absolute posting URL on the tenant's own
* `<slug>.pinpointhq.com` host. It is display-only (written to the
* pipeline and scan history, never server-fetched here), so it is
* not host-locked; the requirement is a well-formed `https:` URL.
* - location: prefer the display `location.name`; otherwise assemble from
* `location.city` / `location.province`.
* - company: the source carries no per-posting company name (each feed is a
* single tenant), so it is sourced from the portal entry name,
* mirroring the recruitee provider.
*
* Rows missing a usable title or a valid `https:` URL are dropped — an empty
* URL would corrupt the scanner's URL-based dedup key.
*
* @param {any} json
* @param {string} companyName
* @returns {Array<{title: string, url: string, company: string, location: string}>}
*/
export function parsePinpointResponse(json, companyName) {
const postings = json?.data;
if (!Array.isArray(postings)) return [];
return postings
.map(j => {
const title = typeof j?.title === 'string' ? j.title.trim() : '';
if (!title) return null;
// url: require a well-formed https: URL (display-only; see doc above).
let url = '';
const rawUrl = typeof j?.url === 'string' ? j.url.trim() : '';
if (rawUrl) {
try {
const parsed = new URL(rawUrl);
if (parsed.protocol === 'https:') url = parsed.href;
} catch {
// malformed URL → leave url = '' → dropped below
}
}
if (!url) return null;
// location: prefer the display name, else assemble from city/province.
const loc = j?.location && typeof j.location === 'object' ? j.location : {};
const name = typeof loc.name === 'string' ? loc.name.trim() : '';
const city = typeof loc.city === 'string' ? loc.city.trim() : '';
const province = typeof loc.province === 'string' ? loc.province.trim() : '';
const location = name || [city, province].filter(Boolean).join(', ');
return { title, url, location, company: companyName };
})
.filter(Boolean);
}
+114
View File
@@ -0,0 +1,114 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Recruitee provider — hits the public per-tenant offers API.
// Auto-detects from careers_url pattern `https://<slug>.recruitee.com`.
// Per-tenant subdomains are the variable part — SSRF defence uses a
// regex match on `<safe-slug>.recruitee.com` rather than a static
// allowlist.
const RECRUITEE_HOST_RE = /^[a-z0-9][a-z0-9-]*\.recruitee\.com$/;
function assertRecruiteeUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`recruitee: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`recruitee: URL must use HTTPS: ${url}`);
if (!RECRUITEE_HOST_RE.test(parsed.hostname)) {
throw new Error(`recruitee: untrusted hostname "${parsed.hostname}" — must match <slug>.recruitee.com`);
}
return url;
}
function resolveApiUrl(entry) {
const raw = typeof entry.careers_url === 'string' ? entry.careers_url : '';
if (!raw) return null;
let parsed;
try {
parsed = new URL(raw);
} catch {
return null;
}
if (parsed.protocol !== 'https:') return null;
if (!RECRUITEE_HOST_RE.test(parsed.hostname)) return null;
return `https://${parsed.hostname}/api/offers/`;
}
/** @type {Provider} */
export default {
id: 'recruitee',
detect(entry) {
const apiUrl = resolveApiUrl(entry);
return apiUrl ? { url: apiUrl } : null;
},
async fetch(entry, ctx) {
const apiUrl = resolveApiUrl(entry);
if (!apiUrl) throw new Error(`recruitee: cannot derive API URL for ${entry.name}`);
assertRecruiteeUrl(apiUrl);
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
return parseRecruiteeResponse(json, entry.name);
},
};
/**
* Parse a Recruitee /api/offers/ response. Exported for unit tests.
*
* Recruitee returns:
* { offers: [{ title, careers_url?, url?, city?, country?, remote?, location? }] }
*
* - url: prefer `careers_url`, fall back to `url`. Recruitee tenants commonly
* serve postings on their own custom domain (e.g. `careers.hostaway.com`),
* so this URL is NOT host-locked to `*.recruitee.com`. Unlike the API
* endpoint, the per-offer URL is display-only — it is written to the pipeline
* and scan history but never server-fetched here, so the SSRF rationale does
* not apply. It is sourced from the already-validated tenant API response.
* Requirement: a well-formed `https:` URL; a non-HTTPS or malformed URL is
* dropped (empty string returned per the Job contract).
* - location: prefer the explicit `location` field; else assemble from
* city/country, appending "Remote" when `remote` is true.
*
* @param {any} json
* @param {string} companyName
* @returns {Array<{title: string, url: string, company: string, location: string}>}
*/
export function parseRecruiteeResponse(json, companyName) {
const offers = json?.offers;
if (!Array.isArray(offers)) return [];
return offers.map(j => {
const city = j.city || '';
const country = j.country || '';
const remote = j.remote ? 'Remote' : '';
const location = j.location || [city, country, remote].filter(Boolean).join(', ');
// Resolve offer URL. Recruitee tenants commonly publish postings on their
// own custom domain (e.g. careers.hostaway.com), so the per-offer URL is
// NOT host-locked to *.recruitee.com — it is display-only (recorded in the
// pipeline/history, never server-fetched here) and comes from the already-
// validated tenant API response. Require only a well-formed https: URL;
// a non-https or malformed URL is dropped (empty string per the Job contract).
let url = '';
const rawUrl = j.careers_url || j.url || '';
if (typeof rawUrl === 'string' && rawUrl) {
try {
const parsed = new URL(rawUrl);
if (parsed.protocol === 'https:') {
url = parsed.href;
}
} catch {
// malformed URL → leave url = ''
}
}
return {
title: j.title || '',
url,
location,
company: companyName,
};
});
}
+43
View File
@@ -0,0 +1,43 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// RemoteOK provider — board-wide aggregator feed (https://remoteok.com/api).
// Returns the latest ~100 remote postings as a JSON array; index 0 is a
// {last_updated, legal} metadata object and is skipped. scan.mjs applies the
// configured title_filter / location_filter to the returned rows.
//
// Wire in via a `job_boards:` entry with `provider: remoteok`.
// RemoteOK API ToS asks for a follow link-back when republishing — N/A for
// private scanning, but don't redistribute this feed publicly without it.
const FEED_URL = 'https://remoteok.com/api';
/** @type {Provider} */
export default {
id: 'remoteok',
/**
* Fetches and normalizes postings from the RemoteOK public feed.
* @param {{ name?: string }} entry - The job_boards entry being processed.
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>}
*/
async fetch(entry, ctx) {
// redirect:'error' prevents SSRF via server-side redirects
const data = await ctx.fetchJson(FEED_URL, { redirect: 'error' });
if (!Array.isArray(data)) {
throw new Error(`remoteok: unexpected API response — expected a JSON array, got ${data === null ? 'null' : typeof data}`);
}
return data
.filter(j => j && typeof j === 'object'
&& typeof j.position === 'string' && j.position.trim() !== ''
&& typeof j.url === 'string' && /^https?:\/\//i.test(j.url.trim()))
.map(j => ({
title: j.position.trim(),
url: j.url.trim(),
company: typeof j.company === 'string' && j.company.trim() ? j.company.trim() : (entry.name || 'RemoteOK'),
location: typeof j.location === 'string' ? j.location.trim() : '',
}));
},
};
+42
View File
@@ -0,0 +1,42 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Remotive provider — board-wide aggregator feed
// (https://remotive.com/api/remote-jobs). Returns { jobs: [...] }. The full
// feed (no ?search=) is fetched so scan.mjs's title_filter can gate on the
// configured AI/ML titles; the feed's own ?search= is too narrow (a substring
// match that misses e.g. "ML Engineer").
//
// Wire in via a `job_boards:` entry with `provider: remotive`.
const FEED_URL = 'https://remotive.com/api/remote-jobs';
/** @type {Provider} */
export default {
id: 'remotive',
/**
* Fetches and normalizes postings from the Remotive public feed.
* @param {{ name?: string }} entry - The job_boards entry being processed.
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>}
*/
async fetch(entry, ctx) {
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(FEED_URL, { redirect: 'error' });
if (!json || !Array.isArray(json.jobs)) {
throw new Error(`remotive: unexpected API response — expected { jobs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
}
return json.jobs
.filter(j => j && typeof j === 'object'
&& typeof j.title === 'string' && j.title.trim() !== ''
&& typeof j.url === 'string' && /^https?:\/\//i.test(j.url.trim()))
.map(j => ({
title: j.title.trim(),
url: j.url.trim(),
company: typeof j.company_name === 'string' && j.company_name.trim() ? j.company_name.trim() : (entry.name || 'Remotive'),
location: typeof j.candidate_required_location === 'string' ? j.candidate_required_location.trim() : '',
}));
},
};
+136
View File
@@ -0,0 +1,136 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Rippling provider — hits the public per-tenant ATS board API.
// Auto-detects from a careers_url like `https://ats.rippling.com/<slug>/jobs`
// (the `<slug>` is the first path segment). Rippling's board API is public and
// zero-auth:
// https://api.rippling.com/platform/api/ats/v1/board/<slug>/jobs
// Response shape: a JSON ARRAY of
// { uuid, name, department: { id, label }, url, workLocation: { id, label } }
//
// The careers host (`ats.rippling.com`) and the API host (`api.rippling.com`)
// are both fixed; the per-tenant slug is the only variable part. It is extracted
// from the careers URL path and constrained to a safe token so it cannot inject
// extra path segments or traversal into the API URL.
const CAREERS_HOST = 'ats.rippling.com';
const API_HOST = 'api.rippling.com';
const API_BASE = `https://${API_HOST}/platform/api/ats/v1/board`;
const SLUG_RE = /^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$/;
/**
* Resolve the tenant slug (e.g. `just-appraised-jobs`) from a careers_url.
* Returns null for non-Rippling, malformed, or unsafe-slug URLs.
* @param {import('./_types.js').PortalEntry} entry
*/
function resolveSlug(entry) {
const raw = typeof entry.careers_url === 'string' ? entry.careers_url : '';
if (!raw) return null;
let parsed;
try {
parsed = new URL(raw);
} catch {
return null;
}
if (parsed.protocol !== 'https:') return null;
if (parsed.hostname !== CAREERS_HOST) return null;
const segment = parsed.pathname.split('/').filter(Boolean)[0] || '';
if (!SLUG_RE.test(segment)) return null;
return segment;
}
/** Build the board API URL for a validated slug. */
function apiUrlForSlug(slug) {
return `${API_BASE}/${encodeURIComponent(slug)}/jobs`;
}
/** @param {string} url */
function assertRipplingApiUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`rippling: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`rippling: URL must use HTTPS: ${url}`);
if (parsed.hostname !== API_HOST) {
throw new Error(`rippling: untrusted hostname "${parsed.hostname}" — must be ${API_HOST}`);
}
return url;
}
/** @type {Provider} */
export default {
id: 'rippling',
detect(entry) {
const slug = resolveSlug(entry);
return slug ? { url: apiUrlForSlug(slug) } : null;
},
async fetch(entry, ctx) {
const slug = resolveSlug(entry);
if (!slug) throw new Error(`rippling: cannot derive API URL for ${entry.name}`);
const apiUrl = apiUrlForSlug(slug);
assertRipplingApiUrl(apiUrl);
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
return parseRipplingResponse(json, entry.name);
},
};
/**
* Parse a Rippling board API response. Exported for unit tests.
*
* The response is a top-level JSON ARRAY of postings. Field mapping → the
* normalized Job shape:
* - title: `name`, trimmed (postings without one are dropped).
* - url: `url` — an absolute `https:` posting URL host-locked to
* `ats.rippling.com` (Rippling always serves postings there, so an
* off-host or non-https URL is untrusted and the posting is dropped).
* It is the dedup key and is display-only (written to the
* pipeline/history, never server-fetched here).
* - company: the portal entry name (the feed is per-tenant and carries no
* company field, same as recruitee).
* - location: `workLocation.label` (e.g. "Remote (United States)"); falls back
* to a bare string `workLocation`, else "".
*
* @param {any} json
* @param {string} companyName
* @returns {Array<{title: string, url: string, company: string, location: string}>}
*/
export function parseRipplingResponse(json, companyName) {
if (!Array.isArray(json)) return [];
return json
.map(j => {
const title = typeof j?.name === 'string' ? j.name.trim() : '';
if (!title) return null;
// url must be an absolute https posting link on ats.rippling.com — Rippling
// always serves postings there (no custom-domain case), so an off-host URL
// is untrusted and dropped. url is the dedup key.
let url = '';
const rawUrl = typeof j?.url === 'string' ? j.url.trim() : '';
if (rawUrl) {
try {
const parsed = new URL(rawUrl);
if (parsed.protocol === 'https:' && parsed.hostname === CAREERS_HOST) url = parsed.href;
} catch {
// malformed URL → leave url = '' → dropped below
}
}
if (!url) return null;
const wl = j?.workLocation;
const location =
wl && typeof wl === 'object' && typeof wl.label === 'string'
? wl.label.trim()
: typeof wl === 'string'
? wl.trim()
: '';
return { title, url, location, company: companyName };
})
.filter(Boolean);
}
+131
View File
@@ -0,0 +1,131 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// SmartRecruiters provider — hits the public postings API.
// Auto-detects from careers_url pattern
// `https://(careers|jobs).smartrecruiters.com/<slug>`. A tracked_companies
// entry can also set `provider: smartrecruiters` explicitly to bypass
// detection (useful when the public careers URL is a branded custom domain).
const ALLOWED_SMARTRECRUITERS_HOSTS = new Set(['api.smartrecruiters.com']);
const SR_CAREERS_HOSTS = new Set(['careers.smartrecruiters.com', 'jobs.smartrecruiters.com']);
const SR_PAGE_SIZE = 100;
const SR_MAX_PAGES = 50; // safety cap (5000 postings @ 100/page)
function assertSmartRecruitersUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`smartrecruiters: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`smartrecruiters: URL must use HTTPS: ${url}`);
if (!ALLOWED_SMARTRECRUITERS_HOSTS.has(parsed.hostname)) {
throw new Error(`smartrecruiters: untrusted hostname "${parsed.hostname}" — must be one of: ${[...ALLOWED_SMARTRECRUITERS_HOSTS].join(', ')}`);
}
return url;
}
function resolveSlug(entry) {
// entry.api takes precedence over careers_url (mirrors greenhouse/ashby) so a
// branded page (e.g. https://jobs.continental.com) can stay as careers_url
// while the SmartRecruiters slug is pinned via
// api: https://careers.smartrecruiters.com/<slug> in portals.yml.
for (const raw of [entry.api, entry.careers_url]) {
if (typeof raw !== 'string' || !raw) continue;
let parsed;
try {
parsed = new URL(raw);
} catch {
continue;
}
if (parsed.protocol !== 'https:') continue;
if (!SR_CAREERS_HOSTS.has(parsed.hostname)) continue;
const slug = parsed.pathname.split('/').filter(Boolean)[0];
if (slug) return slug;
}
return null;
}
function buildPostingsUrl(slug, offset = 0) {
return `https://api.smartrecruiters.com/v1/companies/${slug}/postings?limit=${SR_PAGE_SIZE}&offset=${offset}&status=PUBLIC`;
}
function resolveApiUrl(entry) {
const slug = resolveSlug(entry);
return slug ? buildPostingsUrl(slug, 0) : null;
}
/** @type {Provider} */
export default {
id: 'smartrecruiters',
detect(entry) {
const apiUrl = resolveApiUrl(entry);
return apiUrl ? { url: apiUrl } : null;
},
async fetch(entry, ctx) {
const slug = resolveSlug(entry);
if (!slug) throw new Error(`smartrecruiters: cannot derive API URL for ${entry.name}`);
const all = [];
for (let page = 0; page < SR_MAX_PAGES; page++) {
const apiUrl = buildPostingsUrl(slug, page * SR_PAGE_SIZE);
assertSmartRecruitersUrl(apiUrl);
const json = await ctx.fetchJson(apiUrl, { redirect: 'error' });
const parsed = parseSmartRecruitersResponse(json, entry.name);
if (parsed.length === 0) break;
all.push(...parsed);
if (parsed.length < SR_PAGE_SIZE) break; // last page (short)
}
return all;
},
};
/**
* Parse a SmartRecruiters /postings response. Exported for unit tests.
*
* SmartRecruiters returns:
* { content: [{ id, name, ref, location: { fullLocation?, city?, region?, country?, remote? } }] }
*
* - location: prefer `fullLocation`; else assemble from city/region/country
* parts (skipping empties); append "Remote" when `location.remote` is true.
* - url: `j.ref` is an `api.smartrecruiters.com/v1/companies/<slug>/postings/<id>`
* URL — rewrite to the public `jobs.smartrecruiters.com/<slug>/postings/<id>`.
* If `ref` is missing, synthesise a URL from the company slug + posting id.
*
* @param {any} json
* @param {string} companyName
* @returns {Array<{title: string, url: string, company: string, location: string}>}
*/
export function parseSmartRecruitersResponse(json, companyName) {
const items = json?.content;
if (!Array.isArray(items)) return [];
return items.map(j => {
const loc = j.location || {};
const fullLocation = loc.fullLocation || [loc.city, loc.region, loc.country].filter(Boolean).join(', ');
const remote = loc.remote ? 'Remote' : '';
const location = [fullLocation, remote].filter(Boolean).join(', ');
const slugified = (j.name || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
let url = '';
if (typeof j.ref === 'string') {
let parsedRef;
try { parsedRef = new URL(j.ref); } catch { parsedRef = null; }
if (parsedRef
&& parsedRef.protocol === 'https:'
&& parsedRef.hostname === 'api.smartrecruiters.com'
&& parsedRef.pathname.startsWith('/v1/companies/')) {
const restOfPath = parsedRef.pathname.slice('/v1/companies/'.length);
url = `https://jobs.smartrecruiters.com/${restOfPath}`;
}
}
if (!url && j.id) {
const companySlug = (companyName || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
if (companySlug) {
url = `https://jobs.smartrecruiters.com/${companySlug}/${j.id}-${slugified}`;
}
}
return { title: j.name || '', url, location, company: companyName };
});
}
+159
View File
@@ -0,0 +1,159 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// softgarden provider — the hosted job widgets at
// https://{tenant}.softgarden.io/{lang}/widgets/jobs (e.g. RENK:
// renk-group.softgarden.io/de/widgets/jobs). Companies embed this Wicket page
// as an iframe on their branded career sites; fetched directly it is a plain
// server-rendered document that lists EVERY posting — no auth, no JS, no
// pagination observed (filtering happens client-side via AJAX we don't need).
//
// One posting renders as:
// <div class="matchElement" id="job_id_{id}">
// <div class="matchValue date">04.07.26</div>
// <div class="matchValue title">
// <a href="../../job/{id}/{slug}?jobDbPVId=…&l=de">{Title}</a></div>
// <div class="matchValue audience">…</div>
// <div class="matchValue ProjectGeoLocationCity">…
// <span class="location-view-item">{City}</span>…</div>
// </div>
//
// The href is relative to the /{lang}/widgets/ path — we resolve it against
// the widget URL so it lands on {origin}/job/{id}/{slug}?… (verified live).
// The date is the locale's short form (de: D.M.YY with dots; en variants use
// slashes M/D/YY) — same heuristic as the successfactors CSB parser.
const MAX_JOBS = 1000; // cap postings taken from one widget page
// Minimal HTML entity decoder — titles carry named (&amp;) and numeric
// (&#252; / &#xfc;) entities. Mirrors successfactors.mjs / dassault.mjs.
const NAMED_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
/** @param {string} s */
function decodeEntities(s) {
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (m, body) => {
if (body[0] === '#') {
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
return Number.isInteger(code) && code >= 0 && code <= 0x10ffff ? String.fromCodePoint(code) : m;
}
return NAMED_ENTITIES[body.toLowerCase()] ?? m;
});
}
/** @param {string} s */
function clean(s) {
return decodeEntities(s.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
}
/** Resolve the widget URL: explicit /widgets/jobs URLs pass through, any other
* *.softgarden.io URL defaults to the German jobs widget. */
export function resolveWidgetUrl(entry) {
const raw = entry.api || entry.careers_url || '';
let u;
try {
u = new URL(raw);
} catch {
return null;
}
if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
const host = u.host.toLowerCase();
if (host !== 'softgarden.io' && !host.endsWith('.softgarden.io')) return null;
if (/\/widgets\/jobs\/?$/.test(u.pathname)) return `${u.origin}${u.pathname.replace(/\/$/, '')}`;
const lang = (u.pathname.match(/^\/([a-z]{2})(\/|$)/) || [])[1] || 'de';
return `${u.origin}/${lang}/widgets/jobs`;
}
// Widget dates are the locale's SHORT form: de widgets emit D.M.YY with dots,
// en variants M/D/YY with slashes. Separator infers the field order; 2-digit
// years are 20YY. Junk → undefined ("unknown date", never a bogus timestamp).
/** @param {unknown} raw @returns {number | undefined} */
export function parseSoftgardenDate(raw) {
if (typeof raw !== 'string') return undefined;
const s = raw.trim();
let a, b, year;
const slash = s.match(/^(\d{1,2})\/(\d{1,2})\/(\d{2,4})$/);
const dot = s.match(/^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$/);
if (slash) {
[, a, b, year] = slash; // M/D/Y
} else if (dot) {
[, b, a, year] = dot; // D.M.Y → swap to month/day
} else {
return undefined;
}
const month = Number(a);
const day = Number(b);
let y = Number(year);
if (y < 100) y += 2000;
if (month < 1 || month > 12 || day < 1 || day > 31) return undefined;
const ms = Date.UTC(y, month - 1, day);
return Number.isFinite(ms) ? ms : undefined;
}
/**
* Parse the widget page into raw {id, title, url, location, postedAt} records.
* @param {string} html @param {string} widgetUrl
*/
export function parseWidget(html, widgetUrl) {
if (typeof html !== 'string') return [];
const out = [];
const seen = new Set();
const blocks = html.split(/<div class="matchElement" id="job_id_/).slice(1);
for (const block of blocks) {
const id = (block.match(/^(\d+)"/) || [])[1];
if (!id || seen.has(id)) continue;
const linkM = block.match(/<a href="([^"]*\/job\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/);
if (!linkM) continue;
const title = clean(linkM[2]);
if (!title) continue;
let url;
try {
// hrefs are relative to the /{lang}/widgets/ path ("../../job/…").
url = new URL(decodeEntities(linkM[1]), widgetUrl).href;
} catch {
continue;
}
const cities = [];
const cityRe = /class="location-view-item"[^>]*>([\s\S]*?)<\/span>/g;
let cm;
while ((cm = cityRe.exec(block)) !== null) {
const c = clean(cm[1]);
if (c && !cities.includes(c)) cities.push(c);
}
const dateM = block.match(/class="matchValue date"[^>]*>([\s\S]*?)<\/div>/);
seen.add(id);
out.push({
id,
title,
url,
location: cities.join(' / '),
postedAt: parseSoftgardenDate(dateM ? clean(dateM[1]) : undefined),
});
}
return out;
}
/** @type {Provider} */
export default {
id: 'softgarden',
detect(entry) {
const url = entry.api || entry.careers_url || '';
if (typeof url !== 'string') return null;
return resolveWidgetUrl({ api: url }) ? { url } : null;
},
async fetch(entry, ctx) {
const widgetUrl = resolveWidgetUrl(entry);
if (!widgetUrl) throw new Error(`softgarden: cannot resolve widget URL for ${entry.name}`);
const html = await ctx.fetchText(widgetUrl, { headers: { accept: 'text/html' } });
const rows = parseWidget(html, widgetUrl);
const jobs = [];
for (const row of rows) {
const job = { title: row.title, url: row.url, company: entry.name, location: row.location };
if (typeof row.postedAt === 'number') job.postedAt = row.postedAt;
jobs.push(job);
if (jobs.length >= MAX_JOBS) break;
}
return jobs;
},
};
+82
View File
@@ -0,0 +1,82 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// SolidJobs provider — hits the public offers API.
// Auto-detects from careers_url pattern `https://solid.jobs/public-api/offers/<division>`.
//
// Available divisions: it, engineering, marketing, sales, hr, logistics, finances, other
// API docs: https://solid.jobs/public-api/offers/{division}?campaign={campaign}
const ALLOWED_HOSTS = new Set(['solid.jobs']);
/**
* Validates that the provided URL is a trusted SolidJobs API endpoint.
* Enforces HTTPS protocol, strict hostname matching, and required path prefix.
*
* @param {string} url - The URL string to validate.
* @returns {string} The validated URL string.
* @throws {Error} If the URL is malformed, uses non-HTTPS, has an untrusted host, or wrong path.
*/
function assertUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`solidjobs: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`solidjobs: URL must use HTTPS: ${url}`);
if (!ALLOWED_HOSTS.has(parsed.hostname))
throw new Error(`solidjobs: untrusted hostname "${parsed.hostname}" — must be solid.jobs`);
if (!parsed.pathname.startsWith('/public-api/offers/'))
throw new Error(`solidjobs: URL path must start with /public-api/offers/: ${url}`);
return url;
}
/** @type {Provider} */
export default {
id: 'solidjobs',
/**
* Attempts to detect if the provider can handle the given entry by checking the careers_url.
* * @param {{ careers_url?: string, name?: string }} entry - The configuration entry.
* @returns {{url: string} | null} An object with the matched URL, or null if not matched.
*/
detect(entry) {
const url = entry.careers_url || '';
try {
const parsed = new URL(url);
if (parsed.hostname === 'solid.jobs' && parsed.pathname.startsWith('/public-api/offers/'))
return { url };
} catch {}
return null;
},
/**
* Fetches and normalizes job offers from the SolidJobs public API.
* * @param {{ careers_url?: string, name: string }} entry - The configuration entry being processed.
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>} Array of parsed job offers.
*/
async fetch(entry, ctx) {
const url = entry.careers_url;
if (!url) throw new Error('solidjobs: careers_url required');
assertUrl(url);
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(url, { redirect: 'error' });
if (!json || !Array.isArray(json.jobs)) {
throw new Error(`solidjobs: unexpected API response — expected { jobs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`);
}
/** @type {Array<{ title?: string, url?: string, company?: string, locations?: string | string[] }>} */
const jobs = json.jobs;
return jobs
.filter(j => j && typeof j === 'object' && typeof j.url === 'string' && j.url.trim() !== '')
.map(j => ({
title: j.title || '',
url: /** @type {string} */ (j.url || '').trim(),
company: j.company || entry.name,
location: Array.isArray(j.locations) ? j.locations.join(', ') : (typeof j.locations === 'string' ? j.locations : ''),
}));
},
};
+410
View File
@@ -0,0 +1,410 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// SAP SuccessFactors provider — Recruiting Marketing (RMK, ex-jobs2web) career
// sites (Career Site Builder's branded job boards). These are the portals big
// German industrials run at their own domains (jobs.zf.com, jobs.schaeffler.com,
// jobs.hensoldt.net, jobs.sap.com, …) — all served by the same SF RMK backend.
//
// The RMK backend exposes a public, no-auth job-list fragment endpoint:
// GET {origin}/tile-search-results/?startrow={N}
// It returns an HTML fragment (not JSON) of <li class="job-tile job-id-{id}">
// blocks, each carrying data-url (job path), the title link, and — when the
// tenant configures it — a rendered city field. We parse those out; the scanner
// contract only needs {title, url, company, location, postedAt}.
//
// Pagination: startrow is a plain row offset, and page size is tenant-specific
// (Schaeffler serves 100/page, most others 25). We never assume a page size —
// each loop advances startrow by the number of tiles actually returned, and a
// per-id dedup set guards against overlap or a server that ignores the offset.
//
// Detection: branded RMK hosts (jobs.zf.com) contain no "successfactors" string,
// so detect() only auto-claims literal *.successfactors.eu/.com and jobs2web
// URLs. The branded portals are wired up with an explicit `provider:
// successfactors` in portals.yml (which bypasses detect()); `api:` may override
// careers_url when the public careers_url isn't the RMK origin.
//
// RMK carries no posting date in the list fragment, so postedAt is always
// omitted (consumers treat an absent date as "unknown", never as stale).
//
// ── CSB (Career Site Builder) variant ────────────────────────────────────────
// Newer SF tenants run the Career Site Builder "unified" search (jobs2web's
// searchResultsUnify.js). Their /tile-search-results/ endpoint returns only a
// 16-byte empty <!DOCTYPE html> shell — the jobs are loaded client-side from a
// JSON API instead:
// POST {origin}/services/recruiting/v1/jobs
// Content-Type: application/json
// {"keywords":"","locale":"de_DE","location":"","pageNumber":0,"sortBy":"recent"}
// → {"jobSearchResult":[{"response":{id,unifiedStandardTitle,unifiedUrlTitle,
// jobLocationShort:[…],unifiedStandardStart:"6/18/26",…}}], "totalJobs":N}
// 10 results/page; pageNumber is 0-based. The detail URL the site itself builds
// (from searchResultsUnify.js) is {origin}/job/{urlTitle}/{id}-{locale}.
//
// CRITICAL — locale gating: the API only returns a posting under a locale it was
// published in (each record carries `supportedLocales`). The same tenant reports
// wildly different totals per locale — MAN: de_DE=601 but en_US=8; TRATON: en_US
// only; Hexagon: en_GB only (de_DE/en_US both 0). A hardcoded locale would miss
// entire tenants, so we DISCOVER each tenant's advertised locales from the
// language switcher on {origin}/search/ (the `locale=xx_XX` links), query every
// one, and dedup by job id. CSB carries a real posting date, so postedAt is set.
//
// Strategy selection: `sfVariant: csb` on the portal entry forces CSB directly;
// otherwise the RMK tile path runs first and, only if it yields zero postings
// (the empty-shell signature), we fall back to CSB automatically.
const MAX_PAGES = 40; // safety cap on request count (RMK tile pagination)
const MAX_JOBS = 1000; // cap total postings pulled per site (both strategies)
// CSB knobs.
const CSB_PAGE_SIZE = 10; // fixed by the /services/recruiting/v1/jobs API
const CSB_MAX_PAGES_PER_LOCALE = 100; // safety cap per locale (100*10 = 1000 postings)
const CSB_MAX_LOCALES = 16; // guard against a tenant advertising an absurd locale list
// Locales tried when /search/ discovery yields nothing (fetch failed / markup
// changed). de_DE + en_US cover the DACH targets this provider is aimed at.
const CSB_DEFAULT_LOCALES = ['de_DE', 'en_US'];
// Query order: the DACH-relevant, usually job-rich locales first, so a bounded
// probe (max_pages:1) hits real postings early and dedup keeps a sensible URL
// locale for jobs published in several languages.
const CSB_LOCALE_PRIORITY = ['de_DE', 'en_US', 'en_GB', 'en_EN'];
/** @param {import('./_types.js').PortalEntry} entry */
function resolveConfig(entry) {
const raw = entry.api || entry.careers_url || '';
let u;
try {
u = new URL(raw);
} catch {
return null;
}
if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
return {
origin: u.origin,
tileApi: `${u.origin}/tile-search-results/`,
jobBase: u.origin,
jobsApi: `${u.origin}/services/recruiting/v1/jobs`,
searchPage: `${u.origin}/search/`,
};
}
// Minimal HTML entity decoder — titles carry named (&amp;) and numeric
// (&#252; / &#xfc;) entities. We only need the handful that show up in job
// titles; anything else is left as-is.
const NAMED_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
/** @param {string} s */
function decodeEntities(s) {
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (m, body) => {
if (body[0] === '#') {
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
}
return NAMED_ENTITIES[body.toLowerCase()] ?? m;
});
}
/** @param {string} s */
function clean(s) {
return decodeEntities(s.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
}
// RMK job paths encode the city as the leading slug segment:
// /job/{City-Words}-{Title-Words}-{reqCode}/{id}/
// When the tenant doesn't render a dedicated city field we recover the city by
// stripping the title (and trailing code) off the front slug. We anchor on the
// first two title words — a prefix the slug always reproduces verbatim, unlike
// the title's tail where punctuation like "(m/w/d)" gets mangled.
/** @param {string} dataUrl @param {string} title */
export function cityFromSlug(dataUrl, title) {
let path;
try {
path = decodeURIComponent(dataUrl);
} catch {
path = dataUrl;
}
const m = path.match(/\/job\/([^/]+)\//);
if (!m) return '';
const slug = m[1].toLowerCase();
const words = title.toLowerCase().match(/[\p{L}\p{N}]+/gu);
if (!words || !words.length) return '';
// Anchor on the first two title words, but allow ANY run of non-alphanumerics
// between them — in the slug those words may be joined by "-", "-amp-" (a
// decoded "&"), or several hyphens, none of which survive the title's word
// split. A rigid "word-word" anchor would miss "Program & Release" → "program-&amp;-release".
const esc = (w) => w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
let anchorRe;
try {
anchorRe = new RegExp(words.slice(0, 2).map(esc).join('[^\\p{L}\\p{N}]+'), 'u');
} catch {
return '';
}
const hit = slug.match(anchorRe);
if (!hit || hit.index === undefined || hit.index <= 0) return '';
return slug
.slice(0, hit.index)
.split(/[^\p{L}\p{N}]+/u)
.filter(Boolean)
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
.join(' ');
}
// Parse one page fragment into raw {id, title, url, city} records.
// Each posting renders three responsive copies (desktop/tablet/mobile) inside
// one <li>; a single <li> per id means the top-level dedup already collapses
// them, and within a block the first match of each field wins.
/** @param {string} htmlText @param {string} jobBase */
export function parseTiles(htmlText, jobBase) {
const out = [];
const tileRe = /<li class="job-tile job-id-(\d+)\b[\s\S]*?<\/li>/g;
let t;
while ((t = tileRe.exec(htmlText)) !== null) {
const id = t[1];
const block = t[0];
const urlM = block.match(/data-url="([^"]+)"/);
if (!urlM) continue;
const titleM = block.match(/class="jobTitle-link[^"]*"[^>]*>([\s\S]*?)<\/a>/);
const title = titleM ? clean(titleM[1]) : '';
if (!title) continue;
// data-url is an HTML attribute, so a literal "&" in the path arrives as
// &amp;. Decode entities (but not percent-encoding — %28 etc. stays) before
// both URL building and slug parsing, so the "amp" of &amp; can't leak into
// the recovered city.
const path = decodeEntities(urlM[1]);
// Anchor on id="…-section-city-value"> — the value div. The sibling label
// span references the same id via aria-describedby, so a looser match would
// swallow the "City" label text too.
const cityM = block.match(/id="[^"]*-section-city-value">([\s\S]*?)<\/div>/);
const city = cityM ? clean(cityM[1]) : cityFromSlug(path, title);
const url = /^https?:\/\//i.test(path) ? path : jobBase + (path.startsWith('/') ? path : '/' + path);
out.push({ id, title, url, location: city });
}
return out;
}
// ── CSB helpers ──────────────────────────────────────────────────────────────
// Pull the tenant's advertised locales from the {origin}/search/ page. The CSB
// language switcher renders them as `/search/?q=&startrow=0&locale=xx_XX` links
// (HTML-escaped, so `&amp;` — the regex just anchors on `locale=`). Returns a
// de-duped, priority-ordered list; empty when the markup carries none.
/** @param {string} html */
export function extractLocales(html) {
const found = new Set();
const re = /locale=([a-z]{2}_[A-Z]{2})\b/g;
let m;
while ((m = re.exec(html)) !== null) found.add(m[1]);
const all = [...found];
const prio = (loc) => {
const i = CSB_LOCALE_PRIORITY.indexOf(loc);
return i === -1 ? CSB_LOCALE_PRIORITY.length : i;
};
all.sort((a, b) => prio(a) - prio(b) || a.localeCompare(b));
return all.slice(0, CSB_MAX_LOCALES);
}
// CSB posting dates are rendered in the query locale's short format: US tenants
// send M/D/YY with slashes ("6/18/26"), European (de_DE) tenants send D.M.YY
// with dots ("20.11.23"). Infer the field order from the separator, treat a
// 2-digit year as 20YY, and return epoch ms. Anything unexpected yields
// undefined so a consumer sees "unknown date" rather than a bogus timestamp.
/** @param {unknown} raw */
export function parseCsbDate(raw) {
if (typeof raw !== 'string') return undefined;
const s = raw.trim();
let a, b, year;
let slash = s.match(/^(\d{1,2})\/(\d{1,2})\/(\d{2,4})$/);
let dot = s.match(/^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$/);
if (slash) {
// M/D/Y
[, a, b, year] = slash;
} else if (dot) {
// D.M.Y — swap to month/day
[, b, a, year] = dot;
} else {
return undefined;
}
const month = Number(a);
const day = Number(b);
let y = Number(year);
if (y < 100) y += 2000;
if (month < 1 || month > 12 || day < 1 || day > 31) return undefined;
const ms = Date.UTC(y, month - 1, day);
return Number.isFinite(ms) ? ms : undefined;
}
// jobLocationShort is an array of strings like "Karlovy Vary, CZE, 36004<br/>"
// (one per work location). Strip the trailing <br/> and any other markup, drop
// empties, and join multiple locations with " / ".
/** @param {unknown} raw */
export function cleanCsbLocation(raw) {
const parts = Array.isArray(raw) ? raw : raw == null ? [] : [raw];
const cleaned = [];
for (const p of parts) {
const s = clean(String(p));
if (s && !cleaned.includes(s)) cleaned.push(s);
}
return cleaned.join(' / ');
}
// Map one CSB jobs-API response to raw {id, title, url, location, postedAt}.
// Records nest the useful fields under `.response`; a record without an id or a
// title is skipped (can't build a stable dedup key or a meaningful listing).
/** @param {any} json @param {{origin:string}} cfg @param {string} locale */
export function parseCsbJobs(json, cfg, locale) {
const list = Array.isArray(json?.jobSearchResult) ? json.jobSearchResult : [];
const out = [];
for (const item of list) {
const r = item?.response;
if (!r) continue;
const id = r.id != null ? String(r.id) : '';
const title = clean(String(r.unifiedStandardTitle || r.jobTitle || ''));
if (!id || !title) continue;
// The site builds the slug from urlTitle/unifiedUrlTitle. The slug is purely
// cosmetic — the server keys only on the {id}-{locale} tail (a stub slug
// resolves the same posting) — but tenants render it inconsistently: some
// percent-encode it ("Oper%C3%A1tor-…"), others leave HTML entities in
// ("Mergers-&amp;-Acquisitions"). Decode entities, then drop the URL-
// structural chars (& ? #) a raw entity would otherwise inject into the
// path, leaving any existing %XX escapes untouched.
const slug = decodeEntities(String(r.unifiedUrlTitle || r.urlTitle || 'job'))
.replace(/[?#&]+/g, '-')
.replace(/-{2,}/g, '-');
const url = `${cfg.origin}/job/${slug}/${id}-${locale}`;
out.push({
id,
title,
url,
location: cleanCsbLocation(r.jobLocationShort),
postedAt: parseCsbDate(r.unifiedStandardStart),
});
}
return out;
}
/** Resolve the per-locale page cap: positive integer `max_pages`, else default. */
function resolveCsbMaxPages(entry) {
const v = entry?.max_pages;
if (Number.isInteger(v) && v > 0) return Math.min(v, CSB_MAX_PAGES_PER_LOCALE);
return CSB_MAX_PAGES_PER_LOCALE;
}
// CSB strategy: discover locales, paginate the JSON jobs API per locale, dedup
// by id across locales+pages. `total` from the first page bounds pagination;
// an empty/short page also stops the loop.
/** @param {import('./_types.js').PortalEntry} entry @param {any} cfg @param {import('./_types.js').Context} ctx */
async function fetchCsb(entry, cfg, ctx) {
let locales = CSB_DEFAULT_LOCALES;
try {
const html = await ctx.fetchText(cfg.searchPage, { redirect: 'error', headers: { accept: 'text/html' } });
const discovered = extractLocales(html);
if (discovered.length) locales = discovered;
} catch {
// Discovery is best-effort; fall back to the default locale set.
}
const maxPages = resolveCsbMaxPages(entry);
const jobs = [];
const seen = new Set();
for (const locale of locales) {
if (jobs.length >= MAX_JOBS) break;
let total = null;
for (let page = 0; page < maxPages; page++) {
let json;
try {
json = await ctx.fetchJson(cfg.jobsApi, {
method: 'POST',
redirect: 'error',
headers: { 'content-type': 'application/json', accept: 'application/json' },
body: JSON.stringify({ keywords: '', locale, location: '', pageNumber: page, sortBy: 'recent' }),
});
} catch {
break; // this locale failed (e.g. 307 legacy redirect) — try the next
}
if (total === null) {
total = typeof json?.totalJobs === 'number' ? json.totalJobs : null;
}
// Page-fullness checks below use the raw response count, not rows.length:
// parseCsbJobs drops id/title-less records, and a full API page with a
// few dropped records would otherwise look "short" and stop pagination
// before reaching later pages that still hold valid jobs.
const rawCount = Array.isArray(json?.jobSearchResult) ? json.jobSearchResult.length : 0;
if (rawCount === 0) break;
const rows = parseCsbJobs(json, cfg, locale);
for (const row of rows) {
if (seen.has(row.id)) continue;
seen.add(row.id);
const job = { title: row.title, url: row.url, company: entry.name, location: row.location };
if (typeof row.postedAt === 'number') job.postedAt = row.postedAt;
jobs.push(job);
if (jobs.length >= MAX_JOBS) break;
}
if (jobs.length >= MAX_JOBS) break;
// Stop once we've covered the reported total, or on a short page.
if (total !== null && (page + 1) * CSB_PAGE_SIZE >= total) break;
if (rawCount < CSB_PAGE_SIZE) break;
}
}
return jobs;
}
// RMK strategy: the original /tile-search-results/ HTML-fragment scraper.
/** @param {import('./_types.js').PortalEntry} entry @param {any} cfg @param {import('./_types.js').Context} ctx */
async function fetchRmk(entry, cfg, ctx) {
const jobs = [];
const seen = new Set();
let startrow = 0;
for (let page = 0; page < MAX_PAGES; page++) {
const htmlText = await ctx.fetchText(`${cfg.tileApi}?startrow=${startrow}`, {
redirect: 'error',
headers: { accept: 'text/html' },
});
const tiles = parseTiles(htmlText, cfg.jobBase);
if (tiles.length === 0) break;
let fresh = 0;
for (const tile of tiles) {
if (seen.has(tile.id)) continue;
seen.add(tile.id);
fresh++;
jobs.push({
title: tile.title,
url: tile.url,
company: entry.name,
location: tile.location,
});
}
// No new ids this page → server ignored the offset (or we've looped). Stop.
if (fresh === 0) break;
if (jobs.length >= MAX_JOBS) break;
startrow += tiles.length;
}
// The cap is checked between pages, so the last page can overshoot it.
return jobs.slice(0, MAX_JOBS);
}
/** @type {Provider} */
export default {
id: 'successfactors',
detect(entry) {
const url = entry.api || entry.careers_url || '';
if (/successfactors\.(eu|com)|jobs2web\.com/i.test(url)) return { url };
return null;
},
async fetch(entry, ctx) {
const cfg = resolveConfig(entry);
if (!cfg) throw new Error(`successfactors: cannot resolve origin for ${entry.name}`);
// Explicit opt-in goes straight to CSB. RMK tenants (the majority) run the
// tile scraper; only when it comes back empty — the CSB empty-shell
// signature — do we auto-fall back to the JSON API, so an unflagged CSB
// tenant still works and a genuinely-empty RMK board costs one extra probe.
if (String(entry.sfVariant || '').toLowerCase() === 'csb') {
return fetchCsb(entry, cfg, ctx);
}
const rmkJobs = await fetchRmk(entry, cfg, ctx);
if (rmkJobs.length > 0) return rmkJobs;
return fetchCsb(entry, cfg, ctx);
},
};
+200
View File
@@ -0,0 +1,200 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Teamtailor provider — per-tenant public RSS feed.
//
// Every Teamtailor career site exposes a zero-auth XML jobs feed at
// `https://<slug>.teamtailor.com/jobs.rss`. The feed is public and no-auth, so
// it is parsed in-process with the same tiny tag extractor as
// providers/nodesk.mjs / providers/personio.mjs rather than adding an XML
// dependency.
//
// Auto-detects `https://<slug>.teamtailor.com/...` careers URLs. Many tenants
// front the same feed on a branded domain (e.g. careers.acme.com also serves
// /jobs.rss), so an entry with an explicit `provider: teamtailor` may point its
// `careers_url` (or `api:`) at that branded host and it will be used directly.
//
// SSRF stance mirrors the other providers: auto-detection stays pinned to
// `*.teamtailor.com` so untrusted careers URLs can never steer the fetch at an
// arbitrary host; a branded host is honored only when the user opted in with an
// explicit `provider: teamtailor`. Either way the fetch is HTTPS-only with
// `redirect: 'error'`. Job `<link>`s (branded domains) are emitted as-is and
// never fetched.
const TEAMTAILOR_HOST_RE = /^([a-z0-9](?:[a-z0-9-]*[a-z0-9])?)\.teamtailor\.com$/i;
/**
* Validate a feed URL before fetching. Always HTTPS-only. The hostname is
* pinned to `*.teamtailor.com` for auto-detected entries; an explicit
* `provider: teamtailor` entry may use its configured branded host.
* @param {string} url
* @param {{ explicit?: boolean }} [opts]
*/
function assertFeedUrl(url, { explicit = false } = {}) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`teamtailor: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`teamtailor: URL must use HTTPS: ${url}`);
if (!explicit && !TEAMTAILOR_HOST_RE.test(parsed.hostname)) {
throw new Error(`teamtailor: untrusted hostname "${parsed.hostname}" — must be <slug>.teamtailor.com (or set "provider: teamtailor" to use a branded careers domain)`);
}
return url;
}
// Derive the RSS feed URL from a tracked_companies entry by normalizing any
// path on the configured host to /jobs.rss. Auto-detection (explicit=false)
// only claims *.teamtailor.com hosts; an explicit `provider: teamtailor` entry
// (explicit=true) may use a branded careers host. Returns null otherwise.
/**
* @param {import('./_types.js').PortalEntry} entry
* @param {{ explicit?: boolean }} [opts]
*/
function resolveFeedUrl(entry, { explicit = false } = {}) {
const raw = entry?.api || entry?.careers_url || '';
if (typeof raw !== 'string' || !raw) return null;
let parsed;
try {
parsed = new URL(raw);
} catch {
return null;
}
if (parsed.protocol !== 'https:') return null;
if (!explicit && !TEAMTAILOR_HOST_RE.test(parsed.hostname)) return null;
return `https://${parsed.hostname}/jobs.rss`;
}
function fallbackCompany(entry) {
return typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : 'Teamtailor';
}
/** @type {Provider} */
export default {
id: 'teamtailor',
detect(entry) {
// Auto-detection never claims a branded host — only *.teamtailor.com.
const url = resolveFeedUrl(entry, { explicit: false });
return url ? { url } : null;
},
async fetch(entry, ctx) {
const explicit = entry?.provider === 'teamtailor';
const feedUrl = resolveFeedUrl(entry, { explicit });
if (!feedUrl) throw new Error(`teamtailor: cannot derive jobs.rss URL for ${fallbackCompany(entry)}`);
assertFeedUrl(feedUrl, { explicit });
// redirect:'error' prevents SSRF via server-side redirects; combined with
// assertFeedUrl above it keeps the request pinned to the trusted host
// (*.teamtailor.com, or the branded host the user explicitly configured).
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
return parseTeamtailorFeed(text, fallbackCompany(entry));
},
};
function fromCodePoint(cp) {
try {
return String.fromCodePoint(cp);
} catch {
return '';
}
}
// Decode the XML entities that appear in RSS text: numeric (&#38; / &#x27;)
// and the named five. Numeric forms are decoded first; &amp; is decoded LAST
// so a literal "&amp;lt;" yields "&lt;" rather than over-decoding to "<".
function decodeXmlEntities(s) {
return s
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, '&');
}
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
function extractText(inner) {
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
if (cdata) return cdata[1].trim();
return decodeXmlEntities(inner).trim();
}
// Extract the text of the first <tag>...</tag> in a block. Returns '' when
// absent. Tag names may contain a namespace colon (e.g. tt:city).
function tagText(block, tag) {
const m = block.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, 'i'));
return m ? extractText(m[1]) : '';
}
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
// Keep only absolute HTTPS links. Teamtailor job URLs frequently live on a
// branded custom domain, so — unlike the feed host — the link host is not
// pinned; we only require a well-formed https URL.
function cleanUrl(value) {
if (!value) return '';
try {
const parsed = new URL(value.trim());
return parsed.protocol === 'https:' ? parsed.href : '';
} catch {
return '';
}
}
// Build a human location from the first <tt:location> (city, country). Falls
// back to "Remote" when the posting carries no place but is flagged remote.
function resolveLocation(item) {
const city = tagText(item, 'tt:city');
const country = tagText(item, 'tt:country');
const place = [city, country].filter(Boolean).join(', ');
if (place) return place;
const remote = tagText(item, 'remoteStatus').toLowerCase();
return remote === 'fully' || remote === 'temporary' ? 'Remote' : '';
}
/**
* Parse a Teamtailor public RSS jobs feed. Exported for unit tests.
*
* Shape: `<rss><channel><item>...</item>...</channel></rss>`. Each item
* exposes `<title>`, `<link>`, `<pubDate>`, an optional `<remoteStatus>`, and
* a `tt:` locations block (`<tt:city>`, `<tt:country>`). The company is not in
* the item, so the tracked_companies `name` is used.
*
* @param {string} xml - raw RSS feed body
* @param {string} [defaultCompany] - company label for every posting
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
*/
export function parseTeamtailorFeed(xml, defaultCompany = 'Teamtailor') {
if (typeof xml !== 'string') return [];
const company = typeof defaultCompany === 'string' && defaultCompany.trim() ? defaultCompany.trim() : 'Teamtailor';
const jobs = [];
const blocks = xml.match(/<item\b[^>]*>[\s\S]*?<\/item>/gi) || [];
for (const item of blocks) {
const url = cleanUrl(tagText(item, 'link'));
if (!url) continue;
const title = tagText(item, 'title');
if (!title) continue;
const postedAt = toEpochMs(tagText(item, 'pubDate'));
const job = {
title,
company,
location: resolveLocation(item),
url,
};
if (postedAt !== undefined) job.postedAt = postedAt;
jobs.push(job);
}
return jobs;
}
+322
View File
@@ -0,0 +1,322 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Telegram public-channel provider — reads the public web preview at
// https://t.me/s/{channel}, the same lightweight HTML page Telegram serves
// for link-preview embeds. No Bot API, no MTProto login, no credentials:
// pure HTTP + regex parsing, same zero-token shape as every other provider.
//
// Limitations (accepted trade-off for zero-setup):
// - Public channels only (no invite-only groups — those need a logged-in
// MTProto client, a much heavier and more sensitive integration).
// - Returns roughly the ~20 most recent posts per scan (no pagination).
// Fine for periodic scanning as long as the channel doesn't outpace the
// scan interval.
// - Job fields (title/company/location) are heuristically extracted from
// free-form message text — channels have no fixed schema. Expect noise;
// tune title_filter/content_filter per channel same as any other source.
// - When a post links out to the real source (a career page, hh.ru/geekjob.ru
// posting, an inline "Подробнее"/"Откликнуться" button, etc.), that link
// becomes the job's `url` instead of the Telegram post itself — it's
// usually the fuller, more authoritative JD, and it lets the normal
// liveness checker (Apply-control detection) work as intended. A link
// that only opens inside the Telegram client (a t.me URL, including a bot
// deep link like Runello's) is never usable this way and is skipped; the
// post's own /s/ permalink is the fallback when no real link is found.
//
// Portal entry fields:
// channel — Telegram username without '@', e.g. "jsjobru" (required
// unless careers_url is a bare https://t.me/<channel> link,
// which detect() also accepts)
const TRUSTED_HOST = 't.me';
const CHANNEL_RE = /^[A-Za-z][A-Za-z0-9_]{4,31}$/;
const POST_RE = /data-post="([a-zA-Z0-9_]+)\/(\d+)"/g;
// Some channels post digest/roundup messages ("Подборка вакансий недели")
// that link to many jobs rather than describing one — not a postable offer.
// \b doesn't work with Cyrillic (JS regex \b is defined over ASCII \w only),
// so the boundary is a negative lookahead for another letter/digit instead.
const NON_JOB_TITLE_RE = /^(подборка|дайджест|топ[\s-]|итоги|обзор)(?![\p{L}\p{N}])/iu;
/** @param {string} url */
function assertTelegramUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`telegram: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`telegram: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST)
throw new Error(`telegram: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
return url;
}
function channelFromCareersUrl(careersUrl) {
if (typeof careersUrl !== 'string') return '';
try {
const parsed = new URL(careersUrl);
if (parsed.hostname !== 't.me' && parsed.hostname !== 'telegram.me') return '';
const seg = parsed.pathname.replace(/^\//, '').split('/')[0] || '';
return seg;
} catch {
return '';
}
}
function resolveChannel(entry) {
const raw = typeof entry?.channel === 'string' && entry.channel.trim()
? entry.channel.trim().replace(/^@/, '')
: channelFromCareersUrl(entry?.careers_url);
return raw;
}
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
function fromCodePoint(cp) {
try {
return String.fromCodePoint(cp);
} catch {
return '';
}
}
function decodeHtmlEntities(s) {
return s
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
.replace(/&nbsp;/g, ' ')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, '&');
}
// Convert a message-text block's inner HTML (line breaks + inline tags like
// <a>/<b>/<i>/<span> for links/emphasis/hashtags) into plain text.
function htmlBlockToText(html) {
const withBreaks = html
.replace(/<br\s*\/?>/gi, '\n')
.replace(/<\/(p|div)>/gi, '\n');
const stripped = withBreaks.replace(/<[^>]+>/g, '');
return decodeHtmlEntities(stripped).replace(/\n{3,}/g, '\n\n').trim();
}
// First non-empty line, with leading emoji/bullet clutter stripped, as the
// job title heuristic. Channels commonly lead posts with the role name.
// Some channels pack "Title | Location | Company #hashtags url" onto that
// single line instead of using separate lines — a real job title essentially
// never contains a literal " | ", so cutting there discards the rest as noise
// rather than dragging hashtags/links into the title.
function extractTitle(text) {
const firstLine = text.split('\n').map(l => l.trim()).find(l => l.length > 0) || '';
const firstSegment = firstLine.split(/\s\|\s/)[0].trim();
const cleaned = firstSegment.replace(/^[^\p{L}\p{N}]+/u, '').trim();
return (cleaned || firstSegment).slice(0, 200);
}
// Looks for a line that STARTS with "Label:" (RU/EN variants), anchored to
// line-start so a section header like "О компании:" ("About the company:")
// never matches the "компания" label — it starts with "О", not the label
// itself. Returns '' if no such line exists.
function extractField(text, labelAlternatives) {
const re = new RegExp(`^(?:${labelAlternatives.join('|')})\\s*[:\\-–]\\s*(.+)$`, 'im');
const m = text.match(re);
return m ? m[1].trim() : '';
}
// Second-line markers that mean "this is NOT a company name" — location pins,
// salary lines, work-format tags, or a bullet from a multi-job digest post.
// Different channels put wildly different things on line 2; without this
// guard those get misread as a "company".
const NON_COMPANY_LINE2_RE = /^(?:от|до|from|гибрид|удал[её]нн|офис|remote|hybrid|on-?site|🗺|📍|🔹|💵|💰)/i;
// Fallback when there's no explicit "Company:"/"Location:" line: many
// channels follow "{Title}\n{Company}, {City}" — try the second non-empty
// line, split on the first comma. Conservative on purpose: skipped if that
// line looks like another labeled field, a hashtag/mention, a known
// non-company marker (above), too long to be a short "Name, City" pair, or
// truncated mid-sentence by Telegram's own preview ellipsis — all signs it's
// prose (an "About us" blurb), not a clean company/city pair.
function extractCompanyLocationFromLine2(text) {
const lines = text.split('\n').map(l => l.trim()).filter(l => l.length > 0);
const line2 = lines[1];
if (!line2 || line2.includes(':') || /^[#@]/.test(line2)) return { company: '', location: '' };
if (NON_COMPANY_LINE2_RE.test(line2)) return { company: '', location: '' };
if (line2.length > 60 || /[…]\s*$/.test(line2)) return { company: '', location: '' };
const parts = line2.split(',').map(p => p.trim()).filter(Boolean);
if (parts.length === 1) return { company: line2, location: '' };
if (parts.length === 2 && parts[0].length <= 50 && parts[1].length <= 40) {
return { company: parts[0], location: parts[1] };
}
return { company: '', location: '' };
}
// Many posts don't contain the full JD — they link out to the real source
// (a career page, hh.ru/geekjob.ru posting, LinkedIn job, etc.), either as a
// plain hyperlink in the message text or as an inline-keyboard button
// ("Подробнее", "Откликнуться о вакансии"). A t.me/telegram.me href is never
// a usable source — it just opens another Telegram entity, and for some
// channels (e.g. Runello) that's a bot deep link (t.me/RunelloBot/...) that
// only works inside the Telegram client, never a fetchable web page.
function isUsableExternalUrl(href) {
let parsed;
try {
parsed = new URL(href);
} catch {
return false;
}
if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') return false;
const host = parsed.hostname.toLowerCase();
if (host === 't.me' || host === 'telegram.me') return false;
// A bare domain root ("http://company.com/") is almost always the
// company's marketing homepage, not the specific job posting — some
// channels hyperlink the company's NAME to its homepage while the real
// apply flow is bot-only (e.g. Runello). A real posting link always has
// its own path (/careers/56, /vacancy/123, /jobs/view/123, ...).
if (parsed.pathname === '' || parsed.pathname === '/') return false;
return true;
}
// A recruiter's personal "I'm hiring" LinkedIn post (linkedin.com/posts/...)
// is attribution/sharing context, not the job page itself — deprioritize it
// below a real posting link (company career page, ATS, linkedin.com/jobs/view/...).
function isLinkedinShareUrl(href) {
try {
const parsed = new URL(href);
return /(^|\.)linkedin\.com$/i.test(parsed.hostname) && parsed.pathname.startsWith('/posts/');
} catch {
return false;
}
}
// Collects every <a href> in document order — both inline links inside the
// message text and inline-keyboard buttons share the same tag shape, so one
// pass over the whole per-message block catches both. The message's own
// permalink/hashtag-search anchors are filtered out by isUsableExternalUrl
// (t.me host / relative "?q=" href respectively) without special-casing them.
function extractExternalLinks(block) {
const hrefs = [];
for (const m of block.matchAll(/<a\s[^>]*\bhref="([^"]+)"/gi)) {
// Telegram sometimes double-escapes query strings in href attributes
// (`&amp;amp;` for a literal `&`) — a second decode pass resolves that;
// it's a no-op on already-decoded text since there's no `&amp;` left to match.
hrefs.push(decodeHtmlEntities(decodeHtmlEntities(m[1])));
}
return hrefs.filter(isUsableExternalUrl);
}
// First non-LinkedIn-share candidate wins; a LinkedIn share post is only
// used if nothing else was found (it sometimes carries the full JD text
// pasted into the post itself).
function pickSourceUrl(candidates) {
return candidates.find(h => !isLinkedinShareUrl(h)) || candidates[0] || null;
}
// Defensive cap so an unexpected channel format (e.g. a paragraph slipping
// past the label/line2 heuristics) can't blow up a pipeline.md/tracker cell.
function truncate(s, max) {
if (!s || s.length <= max) return s;
const cut = s.slice(0, max);
const lastSpace = cut.lastIndexOf(' ');
return (lastSpace > max * 0.6 ? cut.slice(0, lastSpace) : cut).trim() + '…';
}
/**
* Parse a t.me/s/{channel} preview page into the canonical Job shape.
* Exported for unit tests.
*
* @param {string} html - raw HTML of the preview page
* @param {{channel: string, fallbackCompany?: string}} opts
* @returns {Array<{title: string, url: string, company: string, location: string, description: string, postedAt?: number}>}
*/
export function parseTelegramChannelFeed(html, { channel, fallbackCompany } = {}) {
if (typeof html !== 'string' || !html) return [];
const matches = [...html.matchAll(POST_RE)];
const jobs = [];
for (let i = 0; i < matches.length; i++) {
const match = matches[i];
const postChannel = match[1];
const msgId = match[2];
if (channel && postChannel.toLowerCase() !== channel.toLowerCase()) continue;
const start = match.index ?? 0;
const end = i + 1 < matches.length ? (matches[i + 1].index ?? html.length) : html.length;
const block = html.slice(start, end);
const textMatch = block.match(/<div class="tgme_widget_message_text[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
if (!textMatch) continue;
const text = htmlBlockToText(textMatch[1]);
if (!text) continue;
const title = extractTitle(text);
if (!title || NON_JOB_TITLE_RE.test(title)) continue;
const labeledCompany = extractField(text, ['компани[яи]', 'работодатель', 'company']);
const labeledLocation = extractField(text, ['локаци[яи]', 'город', 'location', 'city']);
const line2Guess = (!labeledCompany || !labeledLocation)
? extractCompanyLocationFromLine2(text)
: { company: '', location: '' };
const company = truncate(labeledCompany || line2Guess.company || fallbackCompany || channel, 100);
const location = truncate(labeledLocation || line2Guess.location, 60);
const dateMatch = block.match(/<time[^>]*\bdatetime="([^"]+)"/i);
const postedAt = dateMatch ? toEpochMs(dateMatch[1]) : undefined;
// The bare https://t.me/{channel}/{id} permalink renders only a generic
// "Contact @channel" app-install interstitial to a logged-out headless
// browser — no message content at all, which trips the liveness
// checker's insufficient-content guard and false-flags every post as
// expired. The /s/ preview variant (same page family scan() reads)
// actually renders the channel content, so it's the fallback of last
// resort when the post itself doesn't link out to a real source.
const telegramPermalink = `https://t.me/s/${postChannel}/${msgId}`;
const sourceUrl = pickSourceUrl(extractExternalLinks(block));
jobs.push({
title,
url: sourceUrl || telegramPermalink,
company,
location,
description: text,
...(postedAt != null ? { postedAt } : {}),
// Traceability back to the originating post once the URL points
// elsewhere — pipeline.md renders this as a labeled `| note: ...` cell.
...(sourceUrl ? { note: `via Telegram: ${telegramPermalink}` } : {}),
});
}
return jobs;
}
/** @type {Provider} */
export default {
id: 'telegram',
detect(entry) {
const channel = channelFromCareersUrl(entry?.careers_url);
if (!channel || !CHANNEL_RE.test(channel)) return null;
return { url: entry.careers_url };
},
async fetch(entry, ctx) {
const channel = resolveChannel(entry);
if (!channel || !CHANNEL_RE.test(channel)) {
throw new Error(`telegram: missing or invalid "channel" (public Telegram username) for "${entry?.name || 'entry'}"`);
}
const feedUrl = assertTelegramUrl(`https://t.me/s/${channel}`);
const html = await ctx.fetchText(feedUrl, { redirect: 'error' });
const fallbackCompany = typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : channel;
return parseTelegramChannelFeed(html, { channel, fallbackCompany });
},
};
+140
View File
@@ -0,0 +1,140 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// The Hub provider — board-wide aggregator feed (Nordic / EU startups):
// https://thehub.io/api/jobs
// Response shape: { docs: [ { id, key, title, company: { name, ... },
// location: { address, locality, country }, absoluteJobUrl, isRemote,
// publishedAt, createdAt, ... } ], total, page, pages, limit }
//
// Paginated 15/page via `?page=N` (1-indexed); the response carries `pages`, so
// iteration is bounded by min(pages, max_pages). Default cap is modest (the
// board is ~1000 jobs / ~67 pages); override with `max_pages` on the entry.
//
// Wire in via a `job_boards:` entry with `provider: thehub`.
const FEED_BASE = 'https://thehub.io/api/jobs';
const TRUSTED_HOST = 'thehub.io';
const PER_PAGE = 15;
const DEFAULT_MAX_PAGES = 3;
const MAX_PAGES_CAP = 67;
/** @param {string} url */
function assertHubUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`thehub: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`thehub: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST) {
throw new Error(`thehub: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
}
return url;
}
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
function resolveMaxPages(entry) {
const v = entry?.max_pages;
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
return DEFAULT_MAX_PAGES;
}
// NaN-safe Date.parse — `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (typeof value !== 'string' || !value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
/**
* Normalize a single The Hub job. Exported for unit tests.
*
* Field mapping → the normalized Job shape:
* - title: `title`, trimmed (postings without one are dropped).
* - url: `absoluteJobUrl` — host-locked to `thehub.io` (The Hub always
* serves its canonical posting page there; the per-posting `link`
* field is an external apply URL and is NOT used). An off-host or
* non-https URL drops the posting. url is the dedup key and is
* display-only (written to the pipeline/history, never fetched here).
* - company: `company.name`, falling back to the portal entry name, then
* "The Hub".
* - location: `location.address`, else assembled from `location.locality` /
* `location.country`; "Remote" is appended when `isRemote` is true.
* - postedAt: `publishedAt` (else `createdAt`) ISO date → epoch ms (omitted
* when absent/unparseable).
*
* @param {any} j
* @param {string} [fallbackCompany]
* @returns {{ title: string, url: string, company: string, location: string, postedAt?: number } | null}
*/
export function normalizeHubJob(j, fallbackCompany) {
if (!j || typeof j !== 'object') return null;
const title = typeof j.title === 'string' ? j.title.trim() : '';
if (!title) return null;
let url = '';
const rawUrl = typeof j.absoluteJobUrl === 'string' ? j.absoluteJobUrl.trim() : '';
if (rawUrl) {
try {
const parsed = new URL(rawUrl);
if (parsed.protocol === 'https:' && parsed.hostname === TRUSTED_HOST) url = parsed.href;
} catch {
// malformed URL → leave url = '' → dropped below
}
}
if (!url) return null;
const company =
j.company && typeof j.company === 'object' && typeof j.company.name === 'string' && j.company.name.trim()
? j.company.name.trim()
: typeof fallbackCompany === 'string' && fallbackCompany.trim()
? fallbackCompany.trim()
: 'The Hub';
const loc = j.location && typeof j.location === 'object' ? j.location : {};
const address = typeof loc.address === 'string' ? loc.address.trim() : '';
const locality = typeof loc.locality === 'string' ? loc.locality.trim() : '';
const country = typeof loc.country === 'string' ? loc.country.trim() : '';
const base = address || [locality, country].filter(Boolean).join(', ');
const location = [base, j.isRemote === true ? 'Remote' : ''].filter(Boolean).join(', ');
/** @type {{ title: string, url: string, company: string, location: string, postedAt?: number }} */
const job = { title, url, company, location };
const postedAt = toEpochMs(j.publishedAt) ?? toEpochMs(j.createdAt);
if (postedAt !== undefined) job.postedAt = postedAt;
return job;
}
/** @type {Provider} */
export default {
id: 'thehub',
async fetch(entry, ctx) {
assertHubUrl(FEED_BASE);
const maxPages = resolveMaxPages(entry);
const fallbackCompany = entry?.name;
const out = [];
for (let page = 1; page <= maxPages; page++) {
const url = `${FEED_BASE}?page=${page}`;
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(url, { redirect: 'error' });
if (!json || !Array.isArray(json.docs)) {
throw new Error(
`thehub: unexpected API response on page ${page} — expected { docs: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`,
);
}
for (const j of json.docs) {
const normalized = normalizeHubJob(j, fallbackCompany);
if (normalized) out.push(normalized);
}
// Stop at the last page: a short page, or page >= the reported total pages.
if (json.docs.length < PER_PAGE) break;
if (Number.isInteger(json.pages) && page >= json.pages) break;
}
return out;
},
};
+85
View File
@@ -0,0 +1,85 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// The Muse provider — public, zero-auth JSON jobs feed.
// Endpoint: https://www.themuse.com/api/public/jobs?page={n}
// Response shape: { results: [...], page: n, page_count: N }
// All pages are fetched sequentially and aggregated before normalizing.
//
// Wire in via a `job_boards:` entry with `provider: themuse`.
const FEED_BASE = 'https://www.themuse.com/api/public/jobs';
const TRUSTED_HOST = 'www.themuse.com';
/** @param {string} url */
function assertMuseUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`themuse: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`themuse: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST) {
throw new Error(`themuse: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
}
return url;
}
/**
* Normalize a single result from the Muse API response. Exported for unit tests.
*
* Field mapping:
* name → title
* refs.landing_page → url
* company.name → company
* locations[0].name → location
*
* Returns null when required fields (title or url) are missing or invalid.
*
* @param {any} j
* @returns {{ title: string, url: string, company: string, location: string } | null}
*/
export function normalizeMuseJob(j) {
if (!j || typeof j !== 'object') return null;
const title = typeof j.name === 'string' ? j.name.trim() : '';
if (!title) return null;
const url = typeof j.refs?.landing_page === 'string' ? j.refs.landing_page.trim() : '';
if (!url || !/^https?:\/\//i.test(url)) return null;
const company =
typeof j.company?.name === 'string' && j.company.name.trim()
? j.company.name.trim()
: 'The Muse';
const location =
Array.isArray(j.locations) && j.locations.length > 0 && typeof j.locations[0]?.name === 'string'
? j.locations[0].name.trim()
: '';
return { title, url, company, location };
}
/** @type {Provider} */
export default {
id: 'themuse',
async fetch(_entry, ctx) {
assertMuseUrl(FEED_BASE);
const allResults = [];
// Fetch page 0 first to discover page_count, then iterate remaining pages.
let pageCount = 1;
for (let page = 0; page < pageCount; page++) {
const url = `${FEED_BASE}?page=${page}`;
// redirect:'error' prevents SSRF via server-side redirects
const json = await ctx.fetchJson(url, { redirect: 'error' });
if (!json || !Array.isArray(json.results)) {
throw new Error(
`themuse: unexpected API response on page ${page} — expected { results: [...] }, got keys: [${json ? Object.keys(json).join(', ') : 'null'}]`,
);
}
if (page === 0 && Number.isInteger(json.page_count) && json.page_count > 1) {
pageCount = Math.min(json.page_count, 100);
}
allResults.push(...json.results);
}
return allResults.map(normalizeMuseJob).filter(Boolean);
},
};
+144
View File
@@ -0,0 +1,144 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// "Работа России" (trudvsem.ru) provider — hits the Russian government's
// public open-data vacancy API (opendata.trudvsem.ru). Aggregates postings
// submitted to the state employment service; no auth/API key required.
//
// Docs: https://trudvsem.ru/opendata/api
//
// Portal entry fields (all optional except `provider`):
// searchKeywords — `text` query param, e.g. "Frontend Vue" (default: "")
// region — region code for the `region` path filter (default: none — all regions)
// pageSize — `limit` per page, API max is 100 (default: 100)
// maxPages — maximum pages to fetch via `offset` (default: 3)
const API_HOST = 'opendata.trudvsem.ru';
const API_BASE = 'https://opendata.trudvsem.ru/api/v1/vacancies';
const DEFAULT_PAGE_SIZE = 100;
const DEFAULT_MAX_PAGES = 3;
/** @param {string} url */
function assertTrudvsemUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`trudvsem: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`trudvsem: URL must use HTTPS: ${url}`);
if (parsed.hostname !== API_HOST)
throw new Error(`trudvsem: untrusted hostname "${parsed.hostname}" — must be ${API_HOST}`);
return url;
}
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
/**
* Parse a single trudvsem API result item into the canonical Job shape.
*
* The API returns results shaped like:
* { results: { vacancies: [ { vacancy: {
* "job-name": "...", vac_url: "https://trudvsem.ru/vacancy/card/...",
* company: { name: "..." }, region: { name: "..." },
* "creation-date": "2026-07-01", date_modify: "2026-07-02T14:32:05+0300"
* } } ] } }
*
* @param {any} item — raw `{ vacancy: {...} }` wrapper from the results array
* @param {string} fallbackCompany — company name fallback from the portal entry
* @returns {{title: string, url: string, company: string, location: string, postedAt: number|undefined}|null}
*/
export function parseTrudvsemItem(item, fallbackCompany) {
const vacancy = item?.vacancy;
if (!vacancy || typeof vacancy !== 'object') return null;
const title = (vacancy['job-name'] || '').trim();
if (!title) return null;
const rawUrl = (vacancy.vac_url || '').trim();
if (!rawUrl) return null;
let url;
try {
const parsed = new URL(rawUrl);
if (parsed.protocol !== 'https:') return null;
url = parsed.href;
} catch {
return null;
}
const company = (vacancy.company?.name || fallbackCompany || '').trim();
const location = (vacancy.region?.name || '').trim();
const postedAt = toEpochMs(vacancy.date_modify || vacancy['creation-date']);
return { title, url, company, location, ...(postedAt != null ? { postedAt } : {}) };
}
/**
* Build the search URL with query parameters.
* @param {{keywords?: string, region?: string, limit?: number, offset?: number}} params
* @returns {string}
*/
function buildSearchUrl({ keywords, region, limit, offset }) {
const base = region ? `${API_BASE}/region/${encodeURIComponent(region)}` : API_BASE;
const url = new URL(base);
if (keywords) url.searchParams.set('text', keywords);
url.searchParams.set('limit', String(limit || DEFAULT_PAGE_SIZE));
url.searchParams.set('offset', String(offset || 0));
return url.href;
}
/** @type {Provider} */
export default {
id: 'trudvsem',
detect(_entry) {
// trudvsem is a job board aggregator, not a company ATS.
// Use `provider: trudvsem` explicitly in portals.yml.
return null;
},
async fetch(entry, ctx) {
const keywords = entry.searchKeywords || '';
const region = entry.region ? String(entry.region) : '';
const pageSize = Number(entry.pageSize) || DEFAULT_PAGE_SIZE;
const maxPages = Number(entry.maxPages) || DEFAULT_MAX_PAGES;
const fallbackCompany = entry.name || '';
const allJobs = [];
for (let page = 0; page < maxPages; page++) {
const offset = page * pageSize;
const searchUrl = buildSearchUrl({ keywords, region, limit: pageSize, offset });
assertTrudvsemUrl(searchUrl);
let json;
try {
json = /** @type {any} */ (await ctx.fetchJson(searchUrl, { redirect: 'error' }));
} catch (err) {
if (page === 0) throw err;
console.error(`trudvsem: page ${page + 1} fetch failed — ${err.message}`);
break;
}
const vacancies = Array.isArray(json?.results?.vacancies) ? json.results.vacancies : [];
if (vacancies.length === 0) break;
for (const item of vacancies) {
const job = parseTrudvsemItem(item, fallbackCompany);
if (job) allJobs.push(job);
}
if (vacancies.length < pageSize) break;
// Be a polite citizen of a free government API
await new Promise(resolve => setTimeout(resolve, 200));
}
return allJobs;
},
};
+155
View File
@@ -0,0 +1,155 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// We Work Remotely provider - board-wide RSS feed
// (https://weworkremotely.com/remote-jobs.rss). The feed is public, no-auth,
// and XML, so it is parsed in-process with the same tiny tag extractor approach
// as providers/personio.mjs rather than adding an XML dependency.
//
// Wire in via a `job_boards:` entry with `provider: weworkremotely`.
const FEED_URL = 'https://weworkremotely.com/remote-jobs.rss';
const TRUSTED_HOST = 'weworkremotely.com';
/** @param {string} url */
function assertWwrUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`weworkremotely: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`weworkremotely: URL must use HTTPS: ${url}`);
if (parsed.hostname !== TRUSTED_HOST) {
throw new Error(`weworkremotely: untrusted hostname "${parsed.hostname}" - must be ${TRUSTED_HOST}`);
}
return url;
}
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
function toEpochMs(value) {
if (!value) return undefined;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? undefined : parsed;
}
function fallbackCompany(entry) {
return typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : 'We Work Remotely';
}
/** @type {Provider} */
export default {
id: 'weworkremotely',
detect(entry) {
return entry?.provider === 'weworkremotely' ? { url: FEED_URL } : null;
},
async fetch(entry, ctx) {
const feedUrl = assertWwrUrl(FEED_URL);
// redirect:'error' prevents SSRF via server-side redirects; combined with
// assertWwrUrl above it keeps the request pinned to weworkremotely.com.
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
return parseWwrFeed(text, fallbackCompany(entry));
},
};
function fromCodePoint(cp) {
try {
return String.fromCodePoint(cp);
} catch {
return '';
}
}
// Decode the XML entities that appear in RSS text: numeric (&#38; / &#x27;)
// and the named five. Numeric forms are decoded first; &amp; is decoded LAST
// so a literal "&amp;lt;" yields "&lt;" rather than over-decoding to "<".
function decodeXmlEntities(s) {
return s
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, '&');
}
// Resolve a tag's inner text: unwrap a CDATA section, else decode entities.
function extractText(inner) {
const cdata = inner.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
if (cdata) return cdata[1].trim();
return decodeXmlEntities(inner).trim();
}
// Extract the text of the first <tag>...</tag> in a block. Returns '' when absent.
function tagText(block, tag) {
const m = block.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, 'i'));
return m ? extractText(m[1]) : '';
}
function cleanUrl(value) {
if (!value) return '';
const trimmed = value.trim();
try {
const parsed = new URL(trimmed);
const host = parsed.hostname.toLowerCase();
const trusted = host === TRUSTED_HOST || host.endsWith(`.${TRUSTED_HOST}`);
return parsed.protocol === 'https:' && trusted ? parsed.href : '';
} catch {
return '';
}
}
function splitTitle(rawTitle, defaultCompany) {
const text = rawTitle.trim();
const colon = text.indexOf(':');
if (colon > 0) {
const company = text.slice(0, colon).trim();
const title = text.slice(colon + 1).trim();
if (company && title) return { company, title };
}
return { company: defaultCompany, title: text };
}
/**
* Parse We Work Remotely's public RSS jobs feed. Exported for unit tests.
*
* Shape: `<rss><channel><item>...</item>...</channel></rss>`, each item
* carrying `<title>`, `<link>`, `<pubDate>`, and often `<region>` plus
* `<category>`. The title is usually "Company: Role"; when that split is
* present, company and title are normalized separately. The RSS `<link>` is
* the dedup key; items without a usable absolute URL are dropped.
*
* @param {string} xml - raw RSS feed body
* @param {string} [defaultCompany] - fallback company for unsplittable titles
* @returns {Array<{title: string, url: string, company: string, location: string, postedAt?: number}>}
*/
export function parseWwrFeed(xml, defaultCompany = 'We Work Remotely') {
if (typeof xml !== 'string') return [];
const fallback = typeof defaultCompany === 'string' && defaultCompany.trim() ? defaultCompany.trim() : 'We Work Remotely';
const jobs = [];
const blocks = xml.match(/<item\b[^>]*>[\s\S]*?<\/item>/gi) || [];
for (const item of blocks) {
const url = cleanUrl(tagText(item, 'link'));
if (!url) continue;
const rawTitle = tagText(item, 'title');
if (!rawTitle) continue;
const { company, title } = splitTitle(rawTitle, fallback);
const location = tagText(item, 'region') || tagText(item, 'category');
jobs.push({
title,
company,
location,
url,
postedAt: toEpochMs(tagText(item, 'pubDate')),
});
}
return jobs;
}
+103
View File
@@ -0,0 +1,103 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Workable provider — hits the public markdown feed at /<slug>/jobs.md.
// Workable's documented JSON API requires an auth token; the markdown feed
// is the only no-auth public surface. Auto-detects from careers_url pattern
// `https://apply.workable.com/<slug>`. A tracked_companies entry can also
// set `provider: workable` explicitly to bypass detection.
const ALLOWED_WORKABLE_HOSTS = new Set(['apply.workable.com']);
function assertWorkableUrl(url) {
let parsed;
try {
parsed = new URL(url);
} catch {
throw new Error(`workable: invalid URL: ${url}`);
}
if (parsed.protocol !== 'https:') throw new Error(`workable: URL must use HTTPS: ${url}`);
if (!ALLOWED_WORKABLE_HOSTS.has(parsed.hostname)) {
throw new Error(`workable: untrusted hostname "${parsed.hostname}" — must be one of: ${[...ALLOWED_WORKABLE_HOSTS].join(', ')}`);
}
return url;
}
function resolveFeedUrl(entry) {
const raw = typeof entry.careers_url === 'string' ? entry.careers_url : '';
if (!raw) return null;
let parsed;
try {
parsed = new URL(raw);
} catch {
return null;
}
if (parsed.protocol !== 'https:') return null;
if (parsed.hostname !== 'apply.workable.com') return null;
const slug = parsed.pathname.split('/').filter(Boolean)[0];
if (!slug) return null;
return `https://apply.workable.com/${slug}/jobs.md`;
}
/** @type {Provider} */
export default {
id: 'workable',
detect(entry) {
const feedUrl = resolveFeedUrl(entry);
return feedUrl ? { url: feedUrl } : null;
},
async fetch(entry, ctx) {
const feedUrl = resolveFeedUrl(entry);
if (!feedUrl) throw new Error(`workable: cannot derive feed URL for ${entry.name}`);
assertWorkableUrl(feedUrl);
// redirect:'error' prevents SSRF via server-side redirects; combined with
// assertWorkableUrl above it guarantees the final hostname stays in the allowlist.
const text = await ctx.fetchText(feedUrl, { redirect: 'error' });
return parseWorkableMarkdown(text, entry.name);
},
};
/**
* Parse Workable's public markdown feed. Exported as a named export for unit
* tests. The feed exposes a table:
* | Title | Department | Location | Type | Salary | Posted | Details |
* where `Details` holds a markdown link
* [View](https://apply.workable.com/<slug>/jobs/view/<id>.md)
* URLs are validated against `https://apply.workable.com/` — off-domain or
* non-HTTPS [View] links are skipped (not emitted).
*
* @param {string} text — markdown body
* @param {string} companyName — value to write into job.company
* @returns {Array<{title: string, url: string, company: string, location: string}>}
*/
export function parseWorkableMarkdown(text, companyName) {
if (typeof text !== 'string') return [];
const jobs = [];
for (const line of text.split('\n')) {
if (!line.startsWith('|') || !line.includes('[View]')) continue;
const cols = line.split('|').map(c => c.trim());
// Cols: ['', title, dept, location, type, salary, posted, '[View](url.md)', '']
if (cols.length < 8) continue;
const title = cols[1];
if (!title || title === 'Title') continue;
const location = cols[3] || '';
const urlMatch = line.match(/\[View\]\(([^)]+)\)/);
let url = urlMatch ? urlMatch[1] : '';
if (url.endsWith('.md')) url = url.slice(0, -3);
if (!url) continue; // skip rows with no resolvable URL (e.g., malformed [View] link)
// Validate the extracted URL — must parse as https://apply.workable.com/...
try {
const parsedUrl = new URL(url);
if (parsedUrl.protocol !== 'https:' || parsedUrl.hostname !== 'apply.workable.com') continue;
url = parsedUrl.href;
} catch {
continue;
}
jobs.push({ title, url, location, company: companyName });
}
return jobs;
}
+307
View File
@@ -0,0 +1,307 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Workday provider — hits the public CXS jobs endpoint (POST, paginated).
// Auto-detects from careers_url pattern
// `https://<tenant>.<instance>.myworkdayjobs.com[/<locale>]/<site>`,
// e.g. https://23andme.wd5.myworkdayjobs.com/23 →
// POST https://23andme.wd5.myworkdayjobs.com/wday/cxs/23andme/23/jobs
//
// Workday only exposes a relative "postedOn" label ("Posted Today",
// "Posted 5 Days Ago", "Posted 30+ Days Ago"); postedAt is derived from it
// and omitted for the unbounded "30+ Days Ago" form.
const PAGE_SIZE = 20;
// Safety cap on pagination — applied regardless of what the upstream reports
// as `total` (or, when `total` is absent, regardless of how many full pages
// keep coming back), so a misbehaving/compromised API can't drive this into
// fetching an unbounded number of pages. Override with `max_pages` on the
// portal entry for a tenant that genuinely exceeds it.
const DEFAULT_MAX_PAGES = 100;
// Hard ceiling even for an explicit override. 1500 pages (30,000 postings)
// covers known large tenants (dollartree: 23,609; oreillyauto: 17,061;
// cvshealth: ~16,800) with headroom — not a completeness guarantee, since a
// company directory this size has no fixed upper bound.
const MAX_PAGES_CAP = 1500;
// Retry policy for transient page failures (429 rate-limit, 5xx, timeouts/aborts).
// Workday's CXS API is fronted by a WAF that rate-limits in bursts; without
// retry, a single 429 silently truncates an entire tenant (e.g. a
// 3,383-posting tenant reduced to 20 jobs on page 2). Non-transient errors
// (4xx other than 429) are not retried — retrying a malformed request just
// wastes the budget.
const MAX_RETRIES = 3;
const RETRY_BASE_DELAY_MS = 500;
const RETRY_MAX_DELAY_MS = 8_000;
// Delay between successive pages *within one tenant's own pagination loop*
// (not between tenants — that's scan-ats-full.mjs's concurrency, a separate
// knob). A burst of same-host requests with zero delay risks Workday's
// WAF-level rate limiting on any tenant that paginates several pages deep
// (large boards like rollsroyce, sec, roche). Only tenants that loop past
// page 1 pay this; no-date-skip and early-stopped tenants never do.
const INTER_PAGE_DELAY_MS = 150;
// Workday returns postings newest-first, so pagination can stop once a
// page's oldest *dated* posting is well past --since — no point paying for
// (and rate-limit-risking) pages that are entirely stale. Only unambiguous
// numeric ages ("Posted N Days Ago", N < 30) count for this; the unbounded
// "30+ Days Ago" bucket never triggers it, so a wide --since (>=30 days)
// simply never early-stops rather than risk a false stop.
//
// The sort isn't perfectly monotonic day-to-day — some tenants (e.g. Adobe)
// return day-labels slightly out of order across consecutive postings ("27
// Days Ago | 26 Days Ago | 27 Days Ago"), roughly 1 day of jitter. The
// margin only needs to clear that; 2 is double it as a plain safety factor,
// not a second measurement.
const EARLY_STOP_MARGIN_MS = 2 * 86_400_000;
/** Resolve the page cap: a positive integer `max_pages` on the entry, capped. */
function resolveMaxPages(entry) {
const v = entry?.max_pages;
if (Number.isInteger(v) && v > 0) return Math.min(v, MAX_PAGES_CAP);
return DEFAULT_MAX_PAGES;
}
function sleep(ms, ctx) {
if (typeof ctx?.sleep === 'function') return ctx.sleep(ms);
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** Parses a `Retry-After` header value (seconds, or an HTTP-date) to ms, or null. */
function parseRetryAfterMs(value) {
if (!value) return null;
const secs = Number(value);
if (Number.isFinite(secs) && secs >= 0) return secs * 1000;
const dateMs = Date.parse(value);
return Number.isFinite(dateMs) ? Math.max(0, dateMs - Date.now()) : null;
}
function isRetryableError(err) {
const status = err?.status;
if (status === 429) return true;
if (typeof status === 'number' && status >= 500) return true;
return status === undefined; // network error / timeout / abort — no status set
}
/** Fetches a single page, retrying transient failures with backoff. */
async function fetchPageWithRetry(ctx, api, opts) {
let lastErr;
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
try {
return await ctx.fetchJson(api, opts);
} catch (err) {
lastErr = err;
if (attempt === MAX_RETRIES || !isRetryableError(err)) throw err;
const backoff = Math.min(RETRY_BASE_DELAY_MS * 2 ** attempt, RETRY_MAX_DELAY_MS);
// A server-supplied Retry-After is honored, but still clamped — an
// unbounded value (hostile or just misconfigured: Retry-After: 86400)
// would otherwise stall this tenant's fetch for as long as the server
// says, defeating the whole point of a bounded backoff.
const retryAfterMs = parseRetryAfterMs(err?.retryAfter);
const delayMs = retryAfterMs !== null ? Math.min(retryAfterMs, RETRY_MAX_DELAY_MS * 4) : (backoff + Math.random() * 250);
await sleep(delayMs, ctx);
}
}
throw lastErr;
}
/** True once a page's oldest unambiguously-dated posting is past the --since window. */
function pageIsPastWindow(pageJobs, sinceMs) {
if (typeof sinceMs !== 'number') return false;
const dated = pageJobs.map((j) => j.postedAt).filter((v) => typeof v === 'number');
if (dated.length === 0) return false;
return Math.min(...dated) < sinceMs - EARLY_STOP_MARGIN_MS;
}
function resolveEndpoint(entry) {
// Try api: first, then careers_url (mirrors greenhouse/ashby), returning the
// first that matches the Workday tenant pattern. This lets a branded page
// (e.g. https://www.ptc.com/en/careers) stay as careers_url while the Workday
// tenant URL is pinned via api: — and, because we fall through on a non-match,
// a non-Workday api: value doesn't shadow a valid careers_url.
for (const url of [entry.api, entry.careers_url]) {
if (typeof url !== 'string' || !url) continue;
const m = url.match(/^https:\/\/([\w-]+)\.(wd[\w-]*)\.myworkdayjobs\.com\/(?:[a-z]{2}-[A-Z]{2}\/)?([^/?#]+)/);
if (!m) continue;
const [, tenant, instance, site] = m;
const origin = `https://${tenant}.${instance}.myworkdayjobs.com`;
return {
api: `${origin}/wday/cxs/${tenant}/${site}/jobs`,
// externalPath is relative to the site, not the host root — without the
// site segment the URL 404s.
jobBase: `${origin}/${site}`,
};
}
return null;
}
function parsePostedOn(label) {
if (!label) return undefined;
if (/posted\s+today/i.test(label)) return Date.now();
if (/posted\s+yesterday/i.test(label)) return Date.now() - 86_400_000;
const m = label.match(/posted\s+(\d+)(\+?)\s*day/i);
if (!m || m[2] === '+') return undefined; // "30+ Days Ago" — unbounded, no usable date
return Date.now() - Number(m[1]) * 86_400_000;
}
// Workday URL path encodes location as /job/{Location-Slug}/{title-slug}.
// Use it as fallback when locationsText is absent (common on some tenants).
function locationFromPath(externalPath) {
const m = String(externalPath || '').match(/\/job\/([^/]+)\//);
if (!m) return '';
let segment;
try { segment = decodeURIComponent(m[1]); } catch { segment = m[1]; }
return segment.replace(/-/g, ' ');
}
export function parseWorkdayResponse(json, entry) {
const ep = resolveEndpoint(entry);
const jobBase = ep?.jobBase || '';
const postings = Array.isArray(json?.jobPostings) ? json.jobPostings : [];
const jobs = [];
for (const j of postings) {
if (j == null) continue;
if (!j.externalPath || !String(j.title || '').trim()) continue;
jobs.push({
title: j.title || '',
url: jobBase + j.externalPath,
company: entry.name,
location: j.locationsText || locationFromPath(j.externalPath),
postedAt: parsePostedOn(j.postedOn),
});
}
return jobs;
}
/** @type {Provider} */
export default {
id: 'workday',
detect(entry) {
const ep = resolveEndpoint(entry);
return ep ? { url: ep.api } : null;
},
async fetch(entry, ctx) {
const ep = resolveEndpoint(entry);
if (!ep) throw new Error(`workday: cannot derive CXS endpoint for ${entry.name}`);
const postOpts = { method: 'POST', redirect: 'error', headers: { 'content-type': 'application/json', accept: 'application/json' } };
const makeBody = (offset) => JSON.stringify({ limit: PAGE_SIZE, offset, searchText: '', appliedFacets: {} });
const sinceMs = typeof ctx?.sinceMs === 'number' ? ctx.sinceMs : null;
const first = await fetchPageWithRetry(ctx, ep.api, { ...postOpts, body: makeBody(0) });
const jobs = parseWorkdayResponse(first, entry);
const total = typeof first?.total === 'number' ? first.total : null;
const firstPostings = Array.isArray(first?.jobPostings) ? first.jobPostings : [];
const maxPages = resolveMaxPages(entry);
// How many pages to fetch in total (including the first, already-fetched
// one): bounded by `total` when the server reports it, always capped at
// maxPages. When `total` is absent, only probe further pages if the first
// one was full — a short first page already means there's nothing more.
let pagesToFetch = total !== null
? Math.min(Math.ceil(total / PAGE_SIZE), maxPages)
: (firstPostings.length >= PAGE_SIZE ? maxPages : 1);
// Honor a context page cap — verify-portals' liveness probe sets
// `ctx.maxPages: 1` so it only needs to know a board is live, not its full
// count. Without this we'd fetch page 0, then request page 1 and trip the
// probe's second-request sentinel; fetchPageWithRetry treats that abort as
// transient and retries it MAX_RETRIES times (with backoff) before giving up
// — noisy in the logs and rude to the tenant. Capping here makes workday a
// "cooperating provider" that stops after one page and reports an exact
// first-page count. Kept separate from `maxPages` so the entry-cap warning
// below (pagesToFetch === maxPages) stays quiet. No effect on real scans,
// which don't set ctx.maxPages.
const ctxCap = Number.isInteger(ctx?.maxPages) && ctx.maxPages > 0 ? ctx.maxPages : Infinity;
pagesToFetch = Math.min(pagesToFetch, ctxCap);
// Why pagination stopped — drives which warning (if any) fires below.
// 'fetch-error' must NOT produce the "raise max_pages" advice: that knob
// does nothing for a tenant that died on a rate limit rather than hit the cap.
let stopReason = 'complete';
if (pageIsPastWindow(jobs, sinceMs)) stopReason = 'early-stop';
// Some tenants' CXS responses never include postedOn at all (e.g.
// adventhealth, on every page). Early-stop can't apply then — there's
// no dated posting to recognize as "past the window".
const sawAnyDatedPosting = jobs.some((j) => typeof j.postedAt === 'number');
// Zero dated postings on page 0, --include-undated off, --since-bounded
// scan: further pagination is pure waste — every posting from this
// tenant will be dropped downstream as undated regardless of page count
// (newest-first sort means if the *freshest* postings lack a date, older
// ones will too). Return page 0's results instead of grinding to maxPages.
if (stopReason === 'complete' && sinceMs !== null && ctx?.includeUndated !== true
&& !sawAnyDatedPosting && jobs.length > 0) {
stopReason = 'no-date-skip';
}
// Sequential, not concurrent (mirrors providers/4dayweek.mjs, thehub.mjs,
// arbeitnow.mjs, jibeapply.mjs) — a single tenant's API has no reason to
// receive a burst of parallel requests, and a mid-run failure stops
// cleanly with whatever pages were already gathered instead of
// discarding them (Promise.all would fail the whole batch on one error).
let page = 1;
if (stopReason === 'complete') {
for (; page < pagesToFetch; page++) {
await sleep(INTER_PAGE_DELAY_MS, ctx);
let json;
try {
json = await fetchPageWithRetry(ctx, ep.api, { ...postOpts, body: makeBody(page * PAGE_SIZE) });
} catch (err) {
const jobsSummary = `${jobs.length}${total !== null ? ` of ${total}` : ''} jobs`;
console.error(`⚠️ workday: ${entry.name} truncated at ${page + 1} of ${pagesToFetch} pages after ${MAX_RETRIES + 1} attempts (${jobsSummary}): ${err.message}`);
stopReason = 'fetch-error';
break;
}
const pageJobs = parseWorkdayResponse(json, entry);
jobs.push(...pageJobs);
if (total === null) {
const postings = Array.isArray(json?.jobPostings) ? json.jobPostings : [];
if (postings.length < PAGE_SIZE) break; // short page → last page reached
}
if (pageIsPastWindow(pageJobs, sinceMs)) { stopReason = 'early-stop'; break; }
}
if (stopReason === 'complete' && page === pagesToFetch && pagesToFetch === maxPages) {
stopReason = 'cap';
}
}
// The cap is a safety net, not a working limit — silent by design, but a
// tenant that actually hits it needs to be surfaced, in one short line
// (a full-directory scan can hit this on dozens of tenants).
//
// "raise max_pages" only applies when `entry` is a real portals.yml
// tracked_companies entry (scan.mjs, sinceMs === null). scan-ats-full.mjs's
// reverse scan (the only caller that sets ctx.sinceMs) synthesizes entries
// from the external dataset — there's no portal entry to edit, and no
// fixed cap can guarantee full coverage of an unbounded company
// directory anyway, so there's nothing else to suggest.
if (stopReason === 'cap') {
const jobsSummary = `${jobs.length}${total !== null ? ` of ${total}` : ''} jobs`;
if (sinceMs === null) {
console.error(`⚠️ workday: ${entry.name} truncated at max_pages=${maxPages} (${jobsSummary}) — raise max_pages on this entry for more`);
} else {
// Workday's CXS backend can report `total` as exactly
// maxPages*PAGE_SIZE when the real count is far higher (e.g.
// dickssportinggoods: total=2000, public site lists 7,120; requests
// at offset 2000/4000 return the same first posting as offset 0).
// Flag it, don't explain it here.
const suspectTag = total !== null && total === maxPages * PAGE_SIZE ? ' (total may be Workday-capped, not real)' : '';
console.error(`⚠️ workday: ${entry.name} truncated at ${maxPages} pages (${jobsSummary})${suspectTag}`);
}
}
// 'no-date-skip' hits many tenants in a full-directory scan (a company
// with several Workday sites, like a1group or ashealthnet, triggers it
// once per site) — a console.error per hit would repeat thousands of
// times, so tag the array instead; scan-ats-full.mjs aggregates it into
// one summary line.
if (stopReason === 'no-date-skip') jobs.workdayNoDateSkip = true;
return jobs;
},
};
+40
View File
@@ -0,0 +1,40 @@
// @ts-check
/** @typedef {import('./_types.js').Provider} Provider */
// Working Nomads provider — board-wide aggregator feed
// (https://www.workingnomads.com/api/exposed_jobs/). Returns a JSON array of
// postings; scan.mjs applies the configured title_filter / location_filter.
//
// Wire in via a `job_boards:` entry with `provider: workingnomads`.
const FEED_URL = 'https://www.workingnomads.com/api/exposed_jobs/';
/** @type {Provider} */
export default {
id: 'workingnomads',
/**
* Fetches and normalizes postings from the Working Nomads public feed.
* @param {{ name?: string }} entry - The job_boards entry being processed.
* @param {{ fetchJson: (url: string, opts?: { redirect?: 'error'|'follow'|'manual' }) => Promise<any> }} ctx - HTTP context.
* @returns {Promise<Array<{title: string, url: string, company: string, location: string}>>}
*/
async fetch(entry, ctx) {
// redirect:'error' prevents SSRF via server-side redirects
const data = await ctx.fetchJson(FEED_URL, { redirect: 'error' });
if (!Array.isArray(data)) {
throw new Error(`workingnomads: unexpected API response — expected a JSON array, got ${data === null ? 'null' : typeof data}`);
}
return data
.filter(j => j && typeof j === 'object'
&& typeof j.title === 'string' && j.title.trim() !== ''
&& typeof j.url === 'string' && /^https?:\/\//i.test(j.url.trim()))
.map(j => ({
title: j.title.trim(),
url: j.url.trim(),
company: typeof j.company_name === 'string' && j.company_name.trim() ? j.company_name.trim() : (entry.name || 'Working Nomads'),
location: typeof j.location === 'string' ? j.location.trim() : '',
}));
},
};