init

plugins/apify/_apify.mjs

@@ -0,0 +1,202 @@
+// Apify transport helper — runs pre-built actors and returns dataset items.
+// Used by the Apify provider plugin (plugins/apify/index.mjs).
+//
+// Ported from the `providers/_apify.mjs` contributed by @ageem23 in #693 (with
+// thanks). The only change for the plugin layer: runActor() takes the token as
+// an argument (supplied from the plugin's scoped ctx.env) instead of reading
+// process.env directly. Files prefixed with _ are never discovered as plugins.
+//
+// Uses the async pattern (start → poll → fetch dataset) rather than the
+// long-polling /run-sync-get-dataset-items endpoint, which holds one HTTP
+// connection open for the full actor run and gets cut by some networks /
+// Windows SChannel before Apify can flush the response.
+
+// Egress note: this helper keeps its own fetch (its retry / shared-deadline /
+// abort logic is the whole point and doesn't map cleanly onto ctx.fetch). It
+// self-constrains harder than allowedHosts would — every request is built from
+// this single hardcoded base + a strictly-validated actorId (normalizeActorId),
+// so it can only ever reach api.apify.com. The manifest's allowedHosts mirrors
+// that for doctor/review visibility.
+const APIFY_API_BASE = 'https://api.apify.com/v2';
+const DEFAULT_RUN_TIMEOUT_MS = 180_000;
+const POLL_INTERVAL_MS = 3_000;
+const PER_REQUEST_TIMEOUT_MS = 15_000;
+const CONNECT_RETRY_ATTEMPTS = 3;
+const TERMINAL_STATUSES = new Set(['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT']);
+
+export function hasToken(token = process.env.APIFY_TOKEN) {
+  return Boolean(token);
+}
+
+// Apify accepts both "user/actor" and "user~actor" in URLs; normalize to `~`.
+// Validate strictly so a malformed config can't escape the intended
+// /acts/<actor>/runs path with extra `/`, `..`, `?`, or `#` characters and
+// send our bearer token to an unintended endpoint on api.apify.com.
+const ACTOR_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]*[~/][A-Za-z0-9][A-Za-z0-9_.-]*$/;
+
+export function normalizeActorId(actorId) {
+  if (typeof actorId !== 'string' || !ACTOR_ID_RE.test(actorId)) {
+    throw new Error(
+      `apify: invalid actorId ${JSON.stringify(actorId)}. ` +
+      `Expected "owner/actor" or "owner~actor" with letters, digits, "_", ".", or "-" only.`
+    );
+  }
+  const [owner, name] = actorId.split(/[~/]/, 2);
+  return `${encodeURIComponent(owner)}~${encodeURIComponent(name)}`;
+}
+
+// Apify supports auth via ?token= or Authorization: Bearer. The query-string
+// form leaks the token into HTTP access logs and any error/log line that
+// includes the URL, so always use the header.
+function authHeaders(token) {
+  return { authorization: `Bearer ${token}` };
+}
+
+function sleep(ms) {
+  return new Promise(r => setTimeout(r, ms));
+}
+
+async function fetchJsonOnce(url, init = {}, timeoutMs = PER_REQUEST_TIMEOUT_MS) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(url, { ...init, signal: controller.signal });
+    if (!res.ok) {
+      const text = await res.text().catch(() => '');
+      const err = new Error(`HTTP ${res.status}: ${text.slice(0, 300)}`);
+      err.status = res.status;
+      throw err;
+    }
+    return await res.json();
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+// Cold connects to api.apify.com occasionally exceed Undici's 10s internal
+// connect timeout on this host — retry with backoff before giving up. When
+// `deadline` is provided, each attempt's per-request timeout and the
+// inter-attempt backoff are both capped against the remaining budget.
+async function fetchJson(
+  url,
+  init = {},
+  timeoutMs = PER_REQUEST_TIMEOUT_MS,
+  attempts = CONNECT_RETRY_ATTEMPTS,
+  deadline = null,
+) {
+  let lastErr;
+  for (let i = 0; i < attempts; i++) {
+    let thisTimeout = timeoutMs;
+    if (deadline != null) {
+      const remainingMs = deadline - Date.now();
+      if (remainingMs <= 0) throw lastErr || new Error('Apify request budget exhausted');
+      thisTimeout = Math.min(timeoutMs, remainingMs);
+    }
+    try {
+      return await fetchJsonOnce(url, init, thisTimeout);
+    } catch (err) {
+      lastErr = err;
+      if (err.status >= 400 && err.status < 500) throw err;
+      if (i < attempts - 1) {
+        const backoff = 500 * (i + 1);
+        const sleepMs = deadline != null ? Math.min(backoff, deadline - Date.now()) : backoff;
+        if (sleepMs > 0) await sleep(sleepMs);
+        else if (deadline != null) throw lastErr;
+      }
+    }
+  }
+  throw lastErr;
+}
+
+async function startRun(actorId, input, token, deadline = null) {
+  const url = `${APIFY_API_BASE}/acts/${normalizeActorId(actorId)}/runs`;
+  const body = await fetchJson(
+    url,
+    {
+      method: 'POST',
+      headers: { 'content-type': 'application/json', ...authHeaders(token) },
+      body: JSON.stringify(input || {}),
+    },
+    PER_REQUEST_TIMEOUT_MS,
+    CONNECT_RETRY_ATTEMPTS,
+    deadline,
+  );
+  const runId = body?.data?.id;
+  if (!runId) {
+    throw new Error(`Apify did not return a run id: ${JSON.stringify(body).slice(0, 200)}`);
+  }
+  return runId;
+}
+
+// Best-effort — if we give up on a run, stop the actor so credits aren't wasted.
+async function abortRun(runId, token) {
+  const url = `${APIFY_API_BASE}/actor-runs/${runId}/abort`;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), 5_000);
+  try {
+    await fetch(url, { method: 'POST', headers: authHeaders(token), signal: controller.signal });
+  } catch {} finally {
+    clearTimeout(timer);
+  }
+}
+
+async function waitForRun(runId, token, deadline, timeoutMs) {
+  const url = `${APIFY_API_BASE}/actor-runs/${runId}`;
+  let lastError;
+  while (Date.now() < deadline) {
+    const remainingMs = deadline - Date.now();
+    if (remainingMs <= 0) break;
+    try {
+      const body = await fetchJsonOnce(
+        url,
+        { headers: authHeaders(token) },
+        Math.min(PER_REQUEST_TIMEOUT_MS, remainingMs),
+      );
+      const run = body?.data;
+      if (run && TERMINAL_STATUSES.has(run.status)) return run;
+      lastError = undefined;
+    } catch (err) {
+      // 4xx (401/403 auth revoked, 404 run not found) won't succeed on retry.
+      if (err?.status >= 400 && err.status < 500) throw err;
+      lastError = err;
+    }
+    const sleepMs = Math.min(POLL_INTERVAL_MS, deadline - Date.now());
+    if (sleepMs > 0) await sleep(sleepMs);
+  }
+  // Fire-and-forget cleanup; don't add abortRun's 5s to our wall-clock budget.
+  void abortRun(runId, token).catch(() => {});
+  const suffix = lastError ? ` (last error: ${lastError.message})` : '';
+  throw new Error(`Apify run ${runId} did not finish within ${Math.round(timeoutMs / 1000)}s${suffix}`);
+}
+
+async function fetchDatasetItems(runId, token, deadline = null) {
+  const url = `${APIFY_API_BASE}/actor-runs/${runId}/dataset/items`;
+  const items = await fetchJson(
+    url,
+    { headers: authHeaders(token) },
+    PER_REQUEST_TIMEOUT_MS * 2,
+    CONNECT_RETRY_ATTEMPTS,
+    deadline,
+  );
+  if (!Array.isArray(items)) {
+    throw new Error(`Apify run ${runId} returned non-array dataset payload`);
+  }
+  return items;
+}
+
+export async function runActor(actorId, input, { timeoutMs = DEFAULT_RUN_TIMEOUT_MS, token = process.env.APIFY_TOKEN } = {}) {
+  if (!token) throw new Error('APIFY_TOKEN not set');
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
+    throw new Error(`apify: invalid timeoutMs ${JSON.stringify(timeoutMs)} (must be a positive finite number of milliseconds)`);
+  }
+  // Single deadline shared across startRun → waitForRun → fetchDatasetItems so
+  // the caller's timeoutMs is the end-to-end ceiling, not just the wait loop.
+  const deadline = Date.now() + timeoutMs;
+  const runId = await startRun(actorId, input, token, deadline);
+  const run = await waitForRun(runId, token, deadline, timeoutMs);
+  if (run.status !== 'SUCCEEDED') {
+    const reason = run.statusMessage ? `: ${run.statusMessage}` : '';
+    throw new Error(`Apify actor ${actorId} finished with status ${run.status}${reason}`);
+  }
+  return await fetchDatasetItems(runId, token, deadline);
+}

plugins/apify/index.mjs

@@ -0,0 +1,228 @@
+// @ts-check
+// ── Reference seed ── This bundled plugin is a stable, reviewed example. To
+// extend it, publish career-ops-plugin-<id> with "supersedesBundled": true and
+// your version takes precedence once installed (see docs/PLUGINS.md). Bundled
+// seeds take only security/compat fixes — feature work happens in the successor repo.
+//
+// Apify provider plugin — runs any Apify actor and maps its dataset items to
+// the {title, url, company, location} Job shape the scanner expects. All
+// variation (which actor, what input, how to read fields) lives in portals.yml.
+//
+// Ported from the generic Apify provider contributed by @ageem23 in #693 (with
+// thanks); it also homes the LinkedIn-via-Apify use case from #791/#1202. As a
+// KEYED provider it lives here in plugins/ (not the zero-key providers/ dir):
+// enable it in config/plugins.yml and put APIFY_TOKEN in .env. It fires ONLY on
+// a portals.yml entry that sets `provider: apify` — never via auto-detection.
+//
+//   tracked_companies:
+//     - name: "Indeed — VP Engineering (Chicago)"
+//       provider: apify
+//       actor: misceres/indeed-scraper
+//       input: { position: "VP of Engineering", location: "Chicago, IL", maxItems: 25 }
+//       field_map:
+//         title:    [positionName, title]    # array = first non-empty wins
+//         url:      url
+//         company:  [company, companyName]
+//         location: [location, formattedLocation]
+
+import { mkdirSync, writeFileSync, existsSync } from 'fs';
+import { createHash } from 'crypto';
+import { join } from 'path';
+import { hasToken, runActor } from './_apify.mjs';
+
+const JDS_DIR = 'jds';
+const MIN_JD_BODY_CHARS = 50;
+
+function getPath(obj, p) {
+  return p.split('.').reduce((o, k) => (o == null ? undefined : o[k]), obj);
+}
+
+// A valid field_map entry is a single key ('positionName') or an ordered list
+// of fallback keys (['positionName', 'title']). Reject any other shape at
+// config-load time with a clear error instead of crashing mid-scan.
+export function isFieldSpec(spec) {
+  if (typeof spec === 'string') return true;
+  if (Array.isArray(spec) && spec.length > 0 && spec.every(s => typeof s === 'string')) return true;
+  return false;
+}
+
+function pickField(item, spec) {
+  const keys = Array.isArray(spec) ? spec : [spec];
+  for (const k of keys) {
+    const v = getPath(item, k);
+    if (v != null && v !== '') return v;
+  }
+  return '';
+}
+
+const ALLOWED_DEFAULT_KEYS = new Set(['title', 'url', 'company', 'location']);
+
+// Actors return URLs from arbitrary external sites — treat them as untrusted.
+// Reject anything that isn't https so javascript:/data:/file:/http: URLs can't
+// end up clickable in pipeline.md or in the JD-cache filename hash.
+export function isHttpsUrl(value) {
+  try {
+    return new URL(String(value)).protocol === 'https:';
+  } catch {
+    return false;
+  }
+}
+
+function slugify(text) {
+  const slug = String(text || '')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 80);
+  if (slug) return slug;
+  const hash = createHash('sha1').update(String(text || '')).digest('hex').slice(0, 10);
+  return `jd-${hash}`;
+}
+
+function yamlEscape(str) {
+  const s = String(str ?? '').replace(/\n/g, ' ').trim();
+  return `"${s.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
+}
+
+// Lightweight HTML → text for actor description fields (enough for downstream
+// snippet extraction and full evaluation, not a full parser).
+export function htmlToText(s) {
+  const raw = String(s || '');
+  if (!raw || !/[<&]/.test(raw)) return raw.trim();
+  let cleaned = raw
+    .replace(/<script\b[\s\S]*?<\/script\b[^>]*>/gi, ' ')
+    .replace(/<style\b[\s\S]*?<\/style\b[^>]*>/gi, ' ')
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/p>/gi, '\n\n')
+    .replace(/<li[^>]*>/gi, '- ')
+    .replace(/<\/li>/gi, '\n');
+  let prev;
+  do {
+    prev = cleaned;
+    cleaned = cleaned.replace(/<[^>]+>/g, '');
+  } while (cleaned !== prev);
+  return cleaned
+    // Decode &amp; LAST so `&amp;#60;` round-trips to `&#60;` not `<`.
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)))
+    .replace(/&amp;/g, '&')
+    .replace(/[ \t]+/g, ' ')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+// Write jds/{slug}-{hash}.md and return its relative path. The URL-derived hash
+// keeps two distinct postings sharing a company+title from colliding. Atomic
+// (flag:'wx') against the 10-worker TOCTOU race; any FS failure returns null so
+// the caller falls back to the remote URL.
+function saveJd(normalized, descriptionBody, sourceLabel) {
+  let relPath = null;
+  try {
+    mkdirSync(JDS_DIR, { recursive: true });
+    const baseSlug = slugify(`${normalized.company}-${normalized.title}`);
+    const urlHash = createHash('sha1')
+      .update(String(normalized.url || `${normalized.company}-${normalized.title}`))
+      .digest('hex')
+      .slice(0, 10);
+    const filename = `${baseSlug}-${urlHash}.md`;
+    const filepath = join(JDS_DIR, filename);
+    relPath = `${JDS_DIR}/${filename}`;
+    if (existsSync(filepath)) return relPath;
+    const today = new Date().toISOString().slice(0, 10);
+    const content = `---
+title: ${yamlEscape(normalized.title)}
+company: ${yamlEscape(normalized.company)}
+url: ${yamlEscape(normalized.url)}
+location: ${yamlEscape(normalized.location)}
+scraped: "${today}"
+source: ${sourceLabel}
+---
+
+# ${normalized.title} — ${normalized.company}
+
+${descriptionBody}
+`;
+    writeFileSync(filepath, content, { encoding: 'utf-8', flag: 'wx' });
+    return relPath;
+  } catch (err) {
+    if (err?.code === 'EEXIST' && relPath) return relPath;
+    console.warn(`apify: JD cache write failed for ${normalized.title} (${err.code || err.name}: ${err.message}); falling back to remote URL`);
+    return null;
+  }
+}
+
+export function normalizeItem(item, fieldMap, defaults) {
+  const out = {
+    title: String(pickField(item, fieldMap.title) || ''),
+    url: String(pickField(item, fieldMap.url) || ''),
+    company: fieldMap.company ? String(pickField(item, fieldMap.company) || '') : '',
+    location: fieldMap.location ? String(pickField(item, fieldMap.location) || '') : '',
+  };
+  for (const [k, v] of Object.entries(defaults || {})) {
+    if (!ALLOWED_DEFAULT_KEYS.has(k)) continue;
+    if (!out[k]) out[k] = String(v);
+  }
+  return out;
+}
+
+/** The keyed provider hook. Reads APIFY_TOKEN from the plugin's scoped ctx.env. */
+export default {
+  provider: {
+    id: 'apify',
+    // Keyed providers never auto-detect (the engine also forces this to null).
+    detect() { return null; },
+
+    async fetch(entry, ctx) {
+      const token = ctx?.env?.APIFY_TOKEN || process.env.APIFY_TOKEN;
+      if (!hasToken(token)) {
+        throw new Error('APIFY_TOKEN not set — enable apify in config/plugins.yml and add the token to .env');
+      }
+      if (!entry.actor) {
+        throw new Error(`apify: entry ${entry.name} missing 'actor' (e.g. misceres/indeed-scraper)`);
+      }
+      if (
+        !entry.field_map ||
+        !isFieldSpec(entry.field_map.title) ||
+        !isFieldSpec(entry.field_map.url) ||
+        (entry.field_map.company != null && !isFieldSpec(entry.field_map.company)) ||
+        (entry.field_map.location != null && !isFieldSpec(entry.field_map.location)) ||
+        (entry.field_map.description != null && !isFieldSpec(entry.field_map.description))
+      ) {
+        throw new Error(
+          `apify: entry ${entry.name} has invalid field_map. Each of title, url, company, ` +
+          `location, description must be a string or a non-empty array of strings. title and url are required.`
+        );
+      }
+
+      const opts = { token };
+      if (entry.timeout_ms != null) opts.timeoutMs = entry.timeout_ms;
+      const items = await runActor(entry.actor, entry.input || {}, opts);
+
+      const useLocalJd = entry.field_map.description != null;
+      const sourceLabel = String(entry.actor || 'apify').replace(/[^a-z0-9]+/gi, '-').toLowerCase();
+
+      return items
+        .map(item => {
+          const normalized = normalizeItem(item, entry.field_map, entry.defaults);
+          if (!normalized.title || !normalized.url) return null;
+          if (!isHttpsUrl(normalized.url)) return null;
+          if (!useLocalJd) return normalized;
+          const descriptionBody = htmlToText(pickField(item, entry.field_map.description));
+          if (!descriptionBody || descriptionBody.length < MIN_JD_BODY_CHARS) {
+            return normalized;
+          }
+          const remoteUrl = normalized.url;
+          const jdPath = saveJd(normalized, descriptionBody, sourceLabel);
+          if (jdPath === null) return normalized;
+          normalized.url = `local:${jdPath}`;
+          normalized._remote_url = remoteUrl;
+          return normalized;
+        })
+        .filter(j => j && j.title && j.url);
+    },
+  },
+};

plugins/apify/manifest.json

@@ -0,0 +1,13 @@
+{
+  "id": "apify",
+  "name": "Apify",
+  "version": "1.0.0",
+  "apiVersion": 1,
+  "description": "Keyed job source: run an Apify actor and map its results into the scanner.",
+  "hooks": ["provider"],
+  "requiredEnv": ["APIFY_TOKEN"],
+  "allowedHosts": ["api.apify.com"],
+  "skill": "skill.md",
+  "humanInTheLoop": true,
+  "homepage": "https://apify.com"
+}

plugins/apify/skill.md

@@ -0,0 +1,32 @@
+---
+name: career-ops-plugin-apify
+description: How to scan a job source through an Apify actor as a keyed provider.
+license: MIT
+---
+
+# apify plugin
+
+A keyed provider: runs an Apify actor and maps its dataset items into the
+scanner. It fires ONLY on a `portals.yml` entry that sets `provider: apify` —
+never via auto-detection. Put `APIFY_TOKEN` in `.env`.
+
+## portals.yml entry
+
+```yaml
+tracked_companies:
+  - name: "Indeed — VP Engineering (Chicago)"
+    provider: apify
+    actor: misceres/indeed-scraper
+    input: { position: "VP of Engineering", location: "Chicago, IL", maxItems: 25 }
+    field_map:
+      title:    [positionName, title]    # array = first non-empty wins
+      url:      url
+      company:  [company, companyName]
+      location: [location, formattedLocation]
+```
+
+## Then
+
+`node scan.mjs` runs the provider for that entry and writes the results to the
+pipeline like any other source. An optional `field_map.description` caches the
+JD locally under `jds/`.