323 lines
14 KiB
JavaScript
323 lines
14 KiB
JavaScript
// @ts-check
|
||
/** @typedef {import('./_types.js').Provider} Provider */
|
||
|
||
// Telegram public-channel provider — reads the public web preview at
|
||
// https://t.me/s/{channel}, the same lightweight HTML page Telegram serves
|
||
// for link-preview embeds. No Bot API, no MTProto login, no credentials:
|
||
// pure HTTP + regex parsing, same zero-token shape as every other provider.
|
||
//
|
||
// Limitations (accepted trade-off for zero-setup):
|
||
// - Public channels only (no invite-only groups — those need a logged-in
|
||
// MTProto client, a much heavier and more sensitive integration).
|
||
// - Returns roughly the ~20 most recent posts per scan (no pagination).
|
||
// Fine for periodic scanning as long as the channel doesn't outpace the
|
||
// scan interval.
|
||
// - Job fields (title/company/location) are heuristically extracted from
|
||
// free-form message text — channels have no fixed schema. Expect noise;
|
||
// tune title_filter/content_filter per channel same as any other source.
|
||
// - When a post links out to the real source (a career page, hh.ru/geekjob.ru
|
||
// posting, an inline "Подробнее"/"Откликнуться" button, etc.), that link
|
||
// becomes the job's `url` instead of the Telegram post itself — it's
|
||
// usually the fuller, more authoritative JD, and it lets the normal
|
||
// liveness checker (Apply-control detection) work as intended. A link
|
||
// that only opens inside the Telegram client (a t.me URL, including a bot
|
||
// deep link like Runello's) is never usable this way and is skipped; the
|
||
// post's own /s/ permalink is the fallback when no real link is found.
|
||
//
|
||
// Portal entry fields:
|
||
// channel — Telegram username without '@', e.g. "jsjobru" (required
|
||
// unless careers_url is a bare https://t.me/<channel> link,
|
||
// which detect() also accepts)
|
||
|
||
const TRUSTED_HOST = 't.me';
|
||
const CHANNEL_RE = /^[A-Za-z][A-Za-z0-9_]{4,31}$/;
|
||
const POST_RE = /data-post="([a-zA-Z0-9_]+)\/(\d+)"/g;
|
||
|
||
// Some channels post digest/roundup messages ("Подборка вакансий недели")
|
||
// that link to many jobs rather than describing one — not a postable offer.
|
||
// \b doesn't work with Cyrillic (JS regex \b is defined over ASCII \w only),
|
||
// so the boundary is a negative lookahead for another letter/digit instead.
|
||
const NON_JOB_TITLE_RE = /^(подборка|дайджест|топ[\s-]|итоги|обзор)(?![\p{L}\p{N}])/iu;
|
||
|
||
/** @param {string} url */
|
||
function assertTelegramUrl(url) {
|
||
let parsed;
|
||
try {
|
||
parsed = new URL(url);
|
||
} catch {
|
||
throw new Error(`telegram: invalid URL: ${url}`);
|
||
}
|
||
if (parsed.protocol !== 'https:') throw new Error(`telegram: URL must use HTTPS: ${url}`);
|
||
if (parsed.hostname !== TRUSTED_HOST)
|
||
throw new Error(`telegram: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
|
||
return url;
|
||
}
|
||
|
||
function channelFromCareersUrl(careersUrl) {
|
||
if (typeof careersUrl !== 'string') return '';
|
||
try {
|
||
const parsed = new URL(careersUrl);
|
||
if (parsed.hostname !== 't.me' && parsed.hostname !== 'telegram.me') return '';
|
||
const seg = parsed.pathname.replace(/^\//, '').split('/')[0] || '';
|
||
return seg;
|
||
} catch {
|
||
return '';
|
||
}
|
||
}
|
||
|
||
function resolveChannel(entry) {
|
||
const raw = typeof entry?.channel === 'string' && entry.channel.trim()
|
||
? entry.channel.trim().replace(/^@/, '')
|
||
: channelFromCareersUrl(entry?.careers_url);
|
||
return raw;
|
||
}
|
||
|
||
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
|
||
function toEpochMs(value) {
|
||
if (!value) return undefined;
|
||
const parsed = Date.parse(value);
|
||
return Number.isNaN(parsed) ? undefined : parsed;
|
||
}
|
||
|
||
function fromCodePoint(cp) {
|
||
try {
|
||
return String.fromCodePoint(cp);
|
||
} catch {
|
||
return '';
|
||
}
|
||
}
|
||
|
||
function decodeHtmlEntities(s) {
|
||
return s
|
||
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
|
||
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
|
||
.replace(/ /g, ' ')
|
||
.replace(/</g, '<')
|
||
.replace(/>/g, '>')
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, "'")
|
||
.replace(/&/g, '&');
|
||
}
|
||
|
||
// Convert a message-text block's inner HTML (line breaks + inline tags like
|
||
// <a>/<b>/<i>/<span> for links/emphasis/hashtags) into plain text.
|
||
function htmlBlockToText(html) {
|
||
const withBreaks = html
|
||
.replace(/<br\s*\/?>/gi, '\n')
|
||
.replace(/<\/(p|div)>/gi, '\n');
|
||
const stripped = withBreaks.replace(/<[^>]+>/g, '');
|
||
return decodeHtmlEntities(stripped).replace(/\n{3,}/g, '\n\n').trim();
|
||
}
|
||
|
||
// First non-empty line, with leading emoji/bullet clutter stripped, as the
|
||
// job title heuristic. Channels commonly lead posts with the role name.
|
||
// Some channels pack "Title | Location | Company #hashtags url" onto that
|
||
// single line instead of using separate lines — a real job title essentially
|
||
// never contains a literal " | ", so cutting there discards the rest as noise
|
||
// rather than dragging hashtags/links into the title.
|
||
function extractTitle(text) {
|
||
const firstLine = text.split('\n').map(l => l.trim()).find(l => l.length > 0) || '';
|
||
const firstSegment = firstLine.split(/\s\|\s/)[0].trim();
|
||
const cleaned = firstSegment.replace(/^[^\p{L}\p{N}]+/u, '').trim();
|
||
return (cleaned || firstSegment).slice(0, 200);
|
||
}
|
||
|
||
// Looks for a line that STARTS with "Label:" (RU/EN variants), anchored to
|
||
// line-start so a section header like "О компании:" ("About the company:")
|
||
// never matches the "компания" label — it starts with "О", not the label
|
||
// itself. Returns '' if no such line exists.
|
||
function extractField(text, labelAlternatives) {
|
||
const re = new RegExp(`^(?:${labelAlternatives.join('|')})\\s*[:\\-–]\\s*(.+)$`, 'im');
|
||
const m = text.match(re);
|
||
return m ? m[1].trim() : '';
|
||
}
|
||
|
||
// Second-line markers that mean "this is NOT a company name" — location pins,
|
||
// salary lines, work-format tags, or a bullet from a multi-job digest post.
|
||
// Different channels put wildly different things on line 2; without this
|
||
// guard those get misread as a "company".
|
||
const NON_COMPANY_LINE2_RE = /^(?:от|до|from|гибрид|удал[её]нн|офис|remote|hybrid|on-?site|🗺|📍|🔹|💵|💰)/i;
|
||
|
||
// Fallback when there's no explicit "Company:"/"Location:" line: many
|
||
// channels follow "{Title}\n{Company}, {City}" — try the second non-empty
|
||
// line, split on the first comma. Conservative on purpose: skipped if that
|
||
// line looks like another labeled field, a hashtag/mention, a known
|
||
// non-company marker (above), too long to be a short "Name, City" pair, or
|
||
// truncated mid-sentence by Telegram's own preview ellipsis — all signs it's
|
||
// prose (an "About us" blurb), not a clean company/city pair.
|
||
function extractCompanyLocationFromLine2(text) {
|
||
const lines = text.split('\n').map(l => l.trim()).filter(l => l.length > 0);
|
||
const line2 = lines[1];
|
||
if (!line2 || line2.includes(':') || /^[#@]/.test(line2)) return { company: '', location: '' };
|
||
if (NON_COMPANY_LINE2_RE.test(line2)) return { company: '', location: '' };
|
||
if (line2.length > 60 || /[…]\s*$/.test(line2)) return { company: '', location: '' };
|
||
const parts = line2.split(',').map(p => p.trim()).filter(Boolean);
|
||
if (parts.length === 1) return { company: line2, location: '' };
|
||
if (parts.length === 2 && parts[0].length <= 50 && parts[1].length <= 40) {
|
||
return { company: parts[0], location: parts[1] };
|
||
}
|
||
return { company: '', location: '' };
|
||
}
|
||
|
||
// Many posts don't contain the full JD — they link out to the real source
|
||
// (a career page, hh.ru/geekjob.ru posting, LinkedIn job, etc.), either as a
|
||
// plain hyperlink in the message text or as an inline-keyboard button
|
||
// ("Подробнее", "Откликнуться о вакансии"). A t.me/telegram.me href is never
|
||
// a usable source — it just opens another Telegram entity, and for some
|
||
// channels (e.g. Runello) that's a bot deep link (t.me/RunelloBot/...) that
|
||
// only works inside the Telegram client, never a fetchable web page.
|
||
function isUsableExternalUrl(href) {
|
||
let parsed;
|
||
try {
|
||
parsed = new URL(href);
|
||
} catch {
|
||
return false;
|
||
}
|
||
if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') return false;
|
||
const host = parsed.hostname.toLowerCase();
|
||
if (host === 't.me' || host === 'telegram.me') return false;
|
||
// A bare domain root ("http://company.com/") is almost always the
|
||
// company's marketing homepage, not the specific job posting — some
|
||
// channels hyperlink the company's NAME to its homepage while the real
|
||
// apply flow is bot-only (e.g. Runello). A real posting link always has
|
||
// its own path (/careers/56, /vacancy/123, /jobs/view/123, ...).
|
||
if (parsed.pathname === '' || parsed.pathname === '/') return false;
|
||
return true;
|
||
}
|
||
|
||
// A recruiter's personal "I'm hiring" LinkedIn post (linkedin.com/posts/...)
|
||
// is attribution/sharing context, not the job page itself — deprioritize it
|
||
// below a real posting link (company career page, ATS, linkedin.com/jobs/view/...).
|
||
function isLinkedinShareUrl(href) {
|
||
try {
|
||
const parsed = new URL(href);
|
||
return /(^|\.)linkedin\.com$/i.test(parsed.hostname) && parsed.pathname.startsWith('/posts/');
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
// Collects every <a href> in document order — both inline links inside the
|
||
// message text and inline-keyboard buttons share the same tag shape, so one
|
||
// pass over the whole per-message block catches both. The message's own
|
||
// permalink/hashtag-search anchors are filtered out by isUsableExternalUrl
|
||
// (t.me host / relative "?q=" href respectively) without special-casing them.
|
||
function extractExternalLinks(block) {
|
||
const hrefs = [];
|
||
for (const m of block.matchAll(/<a\s[^>]*\bhref="([^"]+)"/gi)) {
|
||
// Telegram sometimes double-escapes query strings in href attributes
|
||
// (`&amp;` for a literal `&`) — a second decode pass resolves that;
|
||
// it's a no-op on already-decoded text since there's no `&` left to match.
|
||
hrefs.push(decodeHtmlEntities(decodeHtmlEntities(m[1])));
|
||
}
|
||
return hrefs.filter(isUsableExternalUrl);
|
||
}
|
||
|
||
// First non-LinkedIn-share candidate wins; a LinkedIn share post is only
|
||
// used if nothing else was found (it sometimes carries the full JD text
|
||
// pasted into the post itself).
|
||
function pickSourceUrl(candidates) {
|
||
return candidates.find(h => !isLinkedinShareUrl(h)) || candidates[0] || null;
|
||
}
|
||
|
||
// Defensive cap so an unexpected channel format (e.g. a paragraph slipping
|
||
// past the label/line2 heuristics) can't blow up a pipeline.md/tracker cell.
|
||
function truncate(s, max) {
|
||
if (!s || s.length <= max) return s;
|
||
const cut = s.slice(0, max);
|
||
const lastSpace = cut.lastIndexOf(' ');
|
||
return (lastSpace > max * 0.6 ? cut.slice(0, lastSpace) : cut).trim() + '…';
|
||
}
|
||
|
||
/**
|
||
* Parse a t.me/s/{channel} preview page into the canonical Job shape.
|
||
* Exported for unit tests.
|
||
*
|
||
* @param {string} html - raw HTML of the preview page
|
||
* @param {{channel: string, fallbackCompany?: string}} opts
|
||
* @returns {Array<{title: string, url: string, company: string, location: string, description: string, postedAt?: number}>}
|
||
*/
|
||
export function parseTelegramChannelFeed(html, { channel, fallbackCompany } = {}) {
|
||
if (typeof html !== 'string' || !html) return [];
|
||
const matches = [...html.matchAll(POST_RE)];
|
||
const jobs = [];
|
||
|
||
for (let i = 0; i < matches.length; i++) {
|
||
const match = matches[i];
|
||
const postChannel = match[1];
|
||
const msgId = match[2];
|
||
if (channel && postChannel.toLowerCase() !== channel.toLowerCase()) continue;
|
||
|
||
const start = match.index ?? 0;
|
||
const end = i + 1 < matches.length ? (matches[i + 1].index ?? html.length) : html.length;
|
||
const block = html.slice(start, end);
|
||
|
||
const textMatch = block.match(/<div class="tgme_widget_message_text[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
||
if (!textMatch) continue;
|
||
const text = htmlBlockToText(textMatch[1]);
|
||
if (!text) continue;
|
||
|
||
const title = extractTitle(text);
|
||
if (!title || NON_JOB_TITLE_RE.test(title)) continue;
|
||
|
||
const labeledCompany = extractField(text, ['компани[яи]', 'работодатель', 'company']);
|
||
const labeledLocation = extractField(text, ['локаци[яи]', 'город', 'location', 'city']);
|
||
const line2Guess = (!labeledCompany || !labeledLocation)
|
||
? extractCompanyLocationFromLine2(text)
|
||
: { company: '', location: '' };
|
||
const company = truncate(labeledCompany || line2Guess.company || fallbackCompany || channel, 100);
|
||
const location = truncate(labeledLocation || line2Guess.location, 60);
|
||
|
||
const dateMatch = block.match(/<time[^>]*\bdatetime="([^"]+)"/i);
|
||
const postedAt = dateMatch ? toEpochMs(dateMatch[1]) : undefined;
|
||
|
||
// The bare https://t.me/{channel}/{id} permalink renders only a generic
|
||
// "Contact @channel" app-install interstitial to a logged-out headless
|
||
// browser — no message content at all, which trips the liveness
|
||
// checker's insufficient-content guard and false-flags every post as
|
||
// expired. The /s/ preview variant (same page family scan() reads)
|
||
// actually renders the channel content, so it's the fallback of last
|
||
// resort when the post itself doesn't link out to a real source.
|
||
const telegramPermalink = `https://t.me/s/${postChannel}/${msgId}`;
|
||
const sourceUrl = pickSourceUrl(extractExternalLinks(block));
|
||
|
||
jobs.push({
|
||
title,
|
||
url: sourceUrl || telegramPermalink,
|
||
company,
|
||
location,
|
||
description: text,
|
||
...(postedAt != null ? { postedAt } : {}),
|
||
// Traceability back to the originating post once the URL points
|
||
// elsewhere — pipeline.md renders this as a labeled `| note: ...` cell.
|
||
...(sourceUrl ? { note: `via Telegram: ${telegramPermalink}` } : {}),
|
||
});
|
||
}
|
||
|
||
return jobs;
|
||
}
|
||
|
||
/** @type {Provider} */
|
||
export default {
|
||
id: 'telegram',
|
||
|
||
detect(entry) {
|
||
const channel = channelFromCareersUrl(entry?.careers_url);
|
||
if (!channel || !CHANNEL_RE.test(channel)) return null;
|
||
return { url: entry.careers_url };
|
||
},
|
||
|
||
async fetch(entry, ctx) {
|
||
const channel = resolveChannel(entry);
|
||
if (!channel || !CHANNEL_RE.test(channel)) {
|
||
throw new Error(`telegram: missing or invalid "channel" (public Telegram username) for "${entry?.name || 'entry'}"`);
|
||
}
|
||
|
||
const feedUrl = assertTelegramUrl(`https://t.me/s/${channel}`);
|
||
const html = await ctx.fetchText(feedUrl, { redirect: 'error' });
|
||
|
||
const fallbackCompany = typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : channel;
|
||
return parseTelegramChannelFeed(html, { channel, fallbackCompany });
|
||
},
|
||
};
|