init
This commit is contained in:
redlineo committed
commit
d295c7f139
1527 files changed
+270726
No files matched your search
@@ -0,0 +1,322 @@
|
||||
// @ts-check
|
||||
/** @typedef {import('./_types.js').Provider} Provider */
|
||||
|
||||
// Telegram public-channel provider — reads the public web preview at
|
||||
// https://t.me/s/{channel}, the same lightweight HTML page Telegram serves
|
||||
// for link-preview embeds. No Bot API, no MTProto login, no credentials:
|
||||
// pure HTTP + regex parsing, same zero-token shape as every other provider.
|
||||
//
|
||||
// Limitations (accepted trade-off for zero-setup):
|
||||
// - Public channels only (no invite-only groups — those need a logged-in
|
||||
// MTProto client, a much heavier and more sensitive integration).
|
||||
// - Returns roughly the ~20 most recent posts per scan (no pagination).
|
||||
// Fine for periodic scanning as long as the channel doesn't outpace the
|
||||
// scan interval.
|
||||
// - Job fields (title/company/location) are heuristically extracted from
|
||||
// free-form message text — channels have no fixed schema. Expect noise;
|
||||
// tune title_filter/content_filter per channel same as any other source.
|
||||
// - When a post links out to the real source (a career page, hh.ru/geekjob.ru
|
||||
// posting, an inline "Подробнее"/"Откликнуться" button, etc.), that link
|
||||
// becomes the job's `url` instead of the Telegram post itself — it's
|
||||
// usually the fuller, more authoritative JD, and it lets the normal
|
||||
// liveness checker (Apply-control detection) work as intended. A link
|
||||
// that only opens inside the Telegram client (a t.me URL, including a bot
|
||||
// deep link like Runello's) is never usable this way and is skipped; the
|
||||
// post's own /s/ permalink is the fallback when no real link is found.
|
||||
//
|
||||
// Portal entry fields:
|
||||
// channel — Telegram username without '@', e.g. "jsjobru" (required
|
||||
// unless careers_url is a bare https://t.me/<channel> link,
|
||||
// which detect() also accepts)
|
||||
|
||||
const TRUSTED_HOST = 't.me';
|
||||
const CHANNEL_RE = /^[A-Za-z][A-Za-z0-9_]{4,31}$/;
|
||||
const POST_RE = /data-post="([a-zA-Z0-9_]+)\/(\d+)"/g;
|
||||
|
||||
// Some channels post digest/roundup messages ("Подборка вакансий недели")
|
||||
// that link to many jobs rather than describing one — not a postable offer.
|
||||
// \b doesn't work with Cyrillic (JS regex \b is defined over ASCII \w only),
|
||||
// so the boundary is a negative lookahead for another letter/digit instead.
|
||||
const NON_JOB_TITLE_RE = /^(подборка|дайджест|топ[\s-]|итоги|обзор)(?![\p{L}\p{N}])/iu;
|
||||
|
||||
/** @param {string} url */
|
||||
function assertTelegramUrl(url) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`telegram: invalid URL: ${url}`);
|
||||
}
|
||||
if (parsed.protocol !== 'https:') throw new Error(`telegram: URL must use HTTPS: ${url}`);
|
||||
if (parsed.hostname !== TRUSTED_HOST)
|
||||
throw new Error(`telegram: untrusted hostname "${parsed.hostname}" — must be ${TRUSTED_HOST}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
function channelFromCareersUrl(careersUrl) {
|
||||
if (typeof careersUrl !== 'string') return '';
|
||||
try {
|
||||
const parsed = new URL(careersUrl);
|
||||
if (parsed.hostname !== 't.me' && parsed.hostname !== 'telegram.me') return '';
|
||||
const seg = parsed.pathname.replace(/^\//, '').split('/')[0] || '';
|
||||
return seg;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function resolveChannel(entry) {
|
||||
const raw = typeof entry?.channel === 'string' && entry.channel.trim()
|
||||
? entry.channel.trim().replace(/^@/, '')
|
||||
: channelFromCareersUrl(entry?.careers_url);
|
||||
return raw;
|
||||
}
|
||||
|
||||
// NaN-safe Date.parse - `|| undefined` would also coerce a valid epoch 0.
|
||||
function toEpochMs(value) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? undefined : parsed;
|
||||
}
|
||||
|
||||
function fromCodePoint(cp) {
|
||||
try {
|
||||
return String.fromCodePoint(cp);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(s) {
|
||||
return s
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => fromCodePoint(parseInt(h, 16)))
|
||||
.replace(/&#(\d+);/g, (_, d) => fromCodePoint(parseInt(d, 10)))
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
// Convert a message-text block's inner HTML (line breaks + inline tags like
|
||||
// <a>/<b>/<i>/<span> for links/emphasis/hashtags) into plain text.
|
||||
function htmlBlockToText(html) {
|
||||
const withBreaks = html
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<\/(p|div)>/gi, '\n');
|
||||
const stripped = withBreaks.replace(/<[^>]+>/g, '');
|
||||
return decodeHtmlEntities(stripped).replace(/\n{3,}/g, '\n\n').trim();
|
||||
}
|
||||
|
||||
// First non-empty line, with leading emoji/bullet clutter stripped, as the
|
||||
// job title heuristic. Channels commonly lead posts with the role name.
|
||||
// Some channels pack "Title | Location | Company #hashtags url" onto that
|
||||
// single line instead of using separate lines — a real job title essentially
|
||||
// never contains a literal " | ", so cutting there discards the rest as noise
|
||||
// rather than dragging hashtags/links into the title.
|
||||
function extractTitle(text) {
|
||||
const firstLine = text.split('\n').map(l => l.trim()).find(l => l.length > 0) || '';
|
||||
const firstSegment = firstLine.split(/\s\|\s/)[0].trim();
|
||||
const cleaned = firstSegment.replace(/^[^\p{L}\p{N}]+/u, '').trim();
|
||||
return (cleaned || firstSegment).slice(0, 200);
|
||||
}
|
||||
|
||||
// Looks for a line that STARTS with "Label:" (RU/EN variants), anchored to
|
||||
// line-start so a section header like "О компании:" ("About the company:")
|
||||
// never matches the "компания" label — it starts with "О", not the label
|
||||
// itself. Returns '' if no such line exists.
|
||||
function extractField(text, labelAlternatives) {
|
||||
const re = new RegExp(`^(?:${labelAlternatives.join('|')})\\s*[:\\-–]\\s*(.+)$`, 'im');
|
||||
const m = text.match(re);
|
||||
return m ? m[1].trim() : '';
|
||||
}
|
||||
|
||||
// Second-line markers that mean "this is NOT a company name" — location pins,
|
||||
// salary lines, work-format tags, or a bullet from a multi-job digest post.
|
||||
// Different channels put wildly different things on line 2; without this
|
||||
// guard those get misread as a "company".
|
||||
const NON_COMPANY_LINE2_RE = /^(?:от|до|from|гибрид|удал[её]нн|офис|remote|hybrid|on-?site|🗺|📍|🔹|💵|💰)/i;
|
||||
|
||||
// Fallback when there's no explicit "Company:"/"Location:" line: many
|
||||
// channels follow "{Title}\n{Company}, {City}" — try the second non-empty
|
||||
// line, split on the first comma. Conservative on purpose: skipped if that
|
||||
// line looks like another labeled field, a hashtag/mention, a known
|
||||
// non-company marker (above), too long to be a short "Name, City" pair, or
|
||||
// truncated mid-sentence by Telegram's own preview ellipsis — all signs it's
|
||||
// prose (an "About us" blurb), not a clean company/city pair.
|
||||
function extractCompanyLocationFromLine2(text) {
|
||||
const lines = text.split('\n').map(l => l.trim()).filter(l => l.length > 0);
|
||||
const line2 = lines[1];
|
||||
if (!line2 || line2.includes(':') || /^[#@]/.test(line2)) return { company: '', location: '' };
|
||||
if (NON_COMPANY_LINE2_RE.test(line2)) return { company: '', location: '' };
|
||||
if (line2.length > 60 || /[…]\s*$/.test(line2)) return { company: '', location: '' };
|
||||
const parts = line2.split(',').map(p => p.trim()).filter(Boolean);
|
||||
if (parts.length === 1) return { company: line2, location: '' };
|
||||
if (parts.length === 2 && parts[0].length <= 50 && parts[1].length <= 40) {
|
||||
return { company: parts[0], location: parts[1] };
|
||||
}
|
||||
return { company: '', location: '' };
|
||||
}
|
||||
|
||||
// Many posts don't contain the full JD — they link out to the real source
|
||||
// (a career page, hh.ru/geekjob.ru posting, LinkedIn job, etc.), either as a
|
||||
// plain hyperlink in the message text or as an inline-keyboard button
|
||||
// ("Подробнее", "Откликнуться о вакансии"). A t.me/telegram.me href is never
|
||||
// a usable source — it just opens another Telegram entity, and for some
|
||||
// channels (e.g. Runello) that's a bot deep link (t.me/RunelloBot/...) that
|
||||
// only works inside the Telegram client, never a fetchable web page.
|
||||
function isUsableExternalUrl(href) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(href);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') return false;
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
if (host === 't.me' || host === 'telegram.me') return false;
|
||||
// A bare domain root ("http://company.com/") is almost always the
|
||||
// company's marketing homepage, not the specific job posting — some
|
||||
// channels hyperlink the company's NAME to its homepage while the real
|
||||
// apply flow is bot-only (e.g. Runello). A real posting link always has
|
||||
// its own path (/careers/56, /vacancy/123, /jobs/view/123, ...).
|
||||
if (parsed.pathname === '' || parsed.pathname === '/') return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// A recruiter's personal "I'm hiring" LinkedIn post (linkedin.com/posts/...)
|
||||
// is attribution/sharing context, not the job page itself — deprioritize it
|
||||
// below a real posting link (company career page, ATS, linkedin.com/jobs/view/...).
|
||||
function isLinkedinShareUrl(href) {
|
||||
try {
|
||||
const parsed = new URL(href);
|
||||
return /(^|\.)linkedin\.com$/i.test(parsed.hostname) && parsed.pathname.startsWith('/posts/');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Collects every <a href> in document order — both inline links inside the
|
||||
// message text and inline-keyboard buttons share the same tag shape, so one
|
||||
// pass over the whole per-message block catches both. The message's own
|
||||
// permalink/hashtag-search anchors are filtered out by isUsableExternalUrl
|
||||
// (t.me host / relative "?q=" href respectively) without special-casing them.
|
||||
function extractExternalLinks(block) {
|
||||
const hrefs = [];
|
||||
for (const m of block.matchAll(/<a\s[^>]*\bhref="([^"]+)"/gi)) {
|
||||
// Telegram sometimes double-escapes query strings in href attributes
|
||||
// (`&amp;` for a literal `&`) — a second decode pass resolves that;
|
||||
// it's a no-op on already-decoded text since there's no `&` left to match.
|
||||
hrefs.push(decodeHtmlEntities(decodeHtmlEntities(m[1])));
|
||||
}
|
||||
return hrefs.filter(isUsableExternalUrl);
|
||||
}
|
||||
|
||||
// First non-LinkedIn-share candidate wins; a LinkedIn share post is only
|
||||
// used if nothing else was found (it sometimes carries the full JD text
|
||||
// pasted into the post itself).
|
||||
function pickSourceUrl(candidates) {
|
||||
return candidates.find(h => !isLinkedinShareUrl(h)) || candidates[0] || null;
|
||||
}
|
||||
|
||||
// Defensive cap so an unexpected channel format (e.g. a paragraph slipping
|
||||
// past the label/line2 heuristics) can't blow up a pipeline.md/tracker cell.
|
||||
function truncate(s, max) {
|
||||
if (!s || s.length <= max) return s;
|
||||
const cut = s.slice(0, max);
|
||||
const lastSpace = cut.lastIndexOf(' ');
|
||||
return (lastSpace > max * 0.6 ? cut.slice(0, lastSpace) : cut).trim() + '…';
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a t.me/s/{channel} preview page into the canonical Job shape.
|
||||
* Exported for unit tests.
|
||||
*
|
||||
* @param {string} html - raw HTML of the preview page
|
||||
* @param {{channel: string, fallbackCompany?: string}} opts
|
||||
* @returns {Array<{title: string, url: string, company: string, location: string, description: string, postedAt?: number}>}
|
||||
*/
|
||||
export function parseTelegramChannelFeed(html, { channel, fallbackCompany } = {}) {
|
||||
if (typeof html !== 'string' || !html) return [];
|
||||
const matches = [...html.matchAll(POST_RE)];
|
||||
const jobs = [];
|
||||
|
||||
for (let i = 0; i < matches.length; i++) {
|
||||
const match = matches[i];
|
||||
const postChannel = match[1];
|
||||
const msgId = match[2];
|
||||
if (channel && postChannel.toLowerCase() !== channel.toLowerCase()) continue;
|
||||
|
||||
const start = match.index ?? 0;
|
||||
const end = i + 1 < matches.length ? (matches[i + 1].index ?? html.length) : html.length;
|
||||
const block = html.slice(start, end);
|
||||
|
||||
const textMatch = block.match(/<div class="tgme_widget_message_text[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
||||
if (!textMatch) continue;
|
||||
const text = htmlBlockToText(textMatch[1]);
|
||||
if (!text) continue;
|
||||
|
||||
const title = extractTitle(text);
|
||||
if (!title || NON_JOB_TITLE_RE.test(title)) continue;
|
||||
|
||||
const labeledCompany = extractField(text, ['компани[яи]', 'работодатель', 'company']);
|
||||
const labeledLocation = extractField(text, ['локаци[яи]', 'город', 'location', 'city']);
|
||||
const line2Guess = (!labeledCompany || !labeledLocation)
|
||||
? extractCompanyLocationFromLine2(text)
|
||||
: { company: '', location: '' };
|
||||
const company = truncate(labeledCompany || line2Guess.company || fallbackCompany || channel, 100);
|
||||
const location = truncate(labeledLocation || line2Guess.location, 60);
|
||||
|
||||
const dateMatch = block.match(/<time[^>]*\bdatetime="([^"]+)"/i);
|
||||
const postedAt = dateMatch ? toEpochMs(dateMatch[1]) : undefined;
|
||||
|
||||
// The bare https://t.me/{channel}/{id} permalink renders only a generic
|
||||
// "Contact @channel" app-install interstitial to a logged-out headless
|
||||
// browser — no message content at all, which trips the liveness
|
||||
// checker's insufficient-content guard and false-flags every post as
|
||||
// expired. The /s/ preview variant (same page family scan() reads)
|
||||
// actually renders the channel content, so it's the fallback of last
|
||||
// resort when the post itself doesn't link out to a real source.
|
||||
const telegramPermalink = `https://t.me/s/${postChannel}/${msgId}`;
|
||||
const sourceUrl = pickSourceUrl(extractExternalLinks(block));
|
||||
|
||||
jobs.push({
|
||||
title,
|
||||
url: sourceUrl || telegramPermalink,
|
||||
company,
|
||||
location,
|
||||
description: text,
|
||||
...(postedAt != null ? { postedAt } : {}),
|
||||
// Traceability back to the originating post once the URL points
|
||||
// elsewhere — pipeline.md renders this as a labeled `| note: ...` cell.
|
||||
...(sourceUrl ? { note: `via Telegram: ${telegramPermalink}` } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
|
||||
/** @type {Provider} */
|
||||
export default {
|
||||
id: 'telegram',
|
||||
|
||||
detect(entry) {
|
||||
const channel = channelFromCareersUrl(entry?.careers_url);
|
||||
if (!channel || !CHANNEL_RE.test(channel)) return null;
|
||||
return { url: entry.careers_url };
|
||||
},
|
||||
|
||||
async fetch(entry, ctx) {
|
||||
const channel = resolveChannel(entry);
|
||||
if (!channel || !CHANNEL_RE.test(channel)) {
|
||||
throw new Error(`telegram: missing or invalid "channel" (public Telegram username) for "${entry?.name || 'entry'}"`);
|
||||
}
|
||||
|
||||
const feedUrl = assertTelegramUrl(`https://t.me/s/${channel}`);
|
||||
const html = await ctx.fetchText(feedUrl, { redirect: 'error' });
|
||||
|
||||
const fallbackCompany = typeof entry?.name === 'string' && entry.name.trim() ? entry.name.trim() : channel;
|
||||
return parseTelegramChannelFeed(html, { channel, fallbackCompany });
|
||||
},
|
||||
};
|
||||
Reference in new issue
Block a user