Fixed multi-job scrape, and added labels and single job scrape

This commit is contained in:
2026-07-10 00:56:26 +00:00
parent 8155b121da
commit a1399b60db
5 changed files with 680 additions and 378 deletions

View File

@@ -1,9 +1,24 @@
const CHECK_INTERVAL = 5; // minutes
const FAST_CHECK_INTERVAL = 5; // minutes
const MULTI_TITLE_INTERVAL = 30; // minutes
const DEBUG = true;
const JOBS_URL = 'https://app.andela.com/';
const PROFILE_URL = 'https://app.andela.com/profile';
const NOTIFICATION_ICON = 'icon128.png';
// Full-Stack must stay first (it's the default we always reset back to).
const PROFESSIONAL_TITLES = [
'Full-Stack Engineer',
'AI Engineer',
'Data Scientist',
'Front-End Engineer',
];
const DEFAULT_TITLE = 'Full-Stack Engineer';
const DEFAULT_NTFY_SERVER = 'https://notify.tstitagency.com';
const DEFAULT_NTFY_TOPIC = 'andela-jobs';
let isLoggingIn = false;
let activeOperation = null; // null | 'fast' | 'sweep' — only one tab operation runs at a time
function log(...args) {
if (DEBUG) console.log('[Andela Notifier]', ...args);
@@ -25,8 +40,8 @@ async function getConfig() {
url: result.appsScriptUrl || '',
key: result.appsScriptKey || '',
email: result.loginEmail || '',
ntfyServer: result.ntfyServer || 'https://notify.tstitagency.com',
ntfyTopic: result.ntfyTopic || '',
ntfyServer: result.ntfyServer || DEFAULT_NTFY_SERVER,
ntfyTopic: result.ntfyTopic || DEFAULT_NTFY_TOPIC,
ntfyToken: result.ntfyToken || '',
});
},
@@ -54,16 +69,44 @@ async function setStoredData(data) {
async function checkIfLoginPage(tabId) {
return new Promise((resolve) => {
chrome.tabs.sendMessage(tabId, { action: 'checkLoginPage' }, (response) => {
if (chrome.runtime.lastError) {
log('Error checking login page:', chrome.runtime.lastError.message);
resolve(false);
} else {
resolve(response?.isLoginPage || false);
}
if (chrome.runtime.lastError) resolve(false);
else resolve(response?.isLoginPage || false);
});
});
}
async function getCurrentProfileTitle(tabId) {
return new Promise((resolve) => {
chrome.tabs.sendMessage(
tabId,
{ action: 'getCurrentProfileTitle' },
(response) => {
if (chrome.runtime.lastError) resolve('');
else resolve((response && response.title) || '');
},
);
});
}
async function setProfileTitle(tabId, title) {
await chrome.tabs.update(tabId, { url: PROFILE_URL });
await new Promise((resolve) => setTimeout(resolve, 3000));
return new Promise((resolve) => {
chrome.tabs.sendMessage(
tabId,
{ action: 'setProfessionalTitle', title: title },
(response) => {
if (chrome.runtime.lastError) {
log('setProfileTitle error:', chrome.runtime.lastError.message);
resolve(false);
} else {
resolve((response && response.success) || false);
}
},
);
});
}
async function handleAutoLogin(tab) {
isLoggingIn = true;
@@ -83,19 +126,13 @@ async function handleAutoLogin(tab) {
return false;
}
log('Step 1: Entering email and clicking login...');
const emailEntered = await new Promise((resolve) => {
chrome.tabs.sendMessage(
tab.id,
{ action: 'enterEmailAndLogin', email: config.email },
(response) => {
if (chrome.runtime.lastError) {
log('Error entering email:', chrome.runtime.lastError.message);
resolve(false);
} else {
resolve(response?.success || false);
}
if (chrome.runtime.lastError) resolve(false);
else resolve(response?.success || false);
},
);
});
@@ -106,8 +143,6 @@ async function handleAutoLogin(tab) {
return false;
}
log('Step 2: Requesting Apps Script to poll for magic link...');
const requestId = 'login-' + Date.now();
const response = await fetch(config.url, {
@@ -128,7 +163,6 @@ async function handleAutoLogin(tab) {
result.error === 'Unauthorized' ||
result.error === 'Unauthorized email'
) {
log('Authentication failed - check your secret key / login email');
chrome.notifications.create('auth-failed', {
type: 'basic',
iconUrl: NOTIFICATION_ICON,
@@ -146,7 +180,6 @@ async function handleAutoLogin(tab) {
return false;
}
log('Step 3: Visiting magic link...');
await chrome.tabs.update(tab.id, { url: result.loginLink });
await new Promise((resolve) => setTimeout(resolve, 5000));
@@ -167,91 +200,143 @@ async function handleAutoLogin(tab) {
}
}
async function fetchJobsViaContentScript() {
log('fetchJobsViaContentScript: Starting...');
// Opens/reuses the Andela tab and handles login if needed. Shared by both
// the fast check and the multi-title sweep.
async function getReadyTab() {
const tabs = await chrome.tabs.query({ url: 'https://app.andela.com/*' });
let tab;
let createdNewTab = false;
try {
const tabs = await chrome.tabs.query({ url: 'https://app.andela.com/*' });
log('Found ' + tabs.length + ' existing Andela tabs');
let tab;
let createdNewTab = false;
if (tabs.length === 0) {
log('Opening new tab for Andela dashboard...');
tab = await chrome.tabs.create({ url: JOBS_URL, active: false });
createdNewTab = true;
await new Promise((resolve) => setTimeout(resolve, 5000));
} else {
tab = tabs[0];
log('Using existing tab with ID: ' + tab.id);
await chrome.tabs.update(tab.id, { url: JOBS_URL });
await new Promise((resolve) => setTimeout(resolve, 4000));
}
const isLoginPage = await checkIfLoginPage(tab.id);
if (isLoginPage) {
log('Session expired! Starting login flow...');
if (isLoggingIn) {
log('Already logging in, skipping...');
if (createdNewTab) chrome.tabs.remove(tab.id).catch(() => {});
return [];
}
const loginSuccess = await handleAutoLogin(tab);
if (!loginSuccess) {
log('Login failed, aborting job check');
if (createdNewTab) chrome.tabs.remove(tab.id).catch(() => {});
return [];
}
log('Login successful, waiting 2 minutes before resuming...');
await new Promise((resolve) => setTimeout(resolve, 120000));
await chrome.tabs.update(tab.id, { url: JOBS_URL });
await new Promise((resolve) => setTimeout(resolve, 4000));
}
let jobs = [];
try {
log('Extracting jobs from dashboard (expanding "See more" first)...');
jobs = await new Promise((resolve) => {
chrome.tabs.sendMessage(
tab.id,
{ action: 'extractJobs' },
(response) => {
if (chrome.runtime.lastError) {
log('Content script error:', chrome.runtime.lastError.message);
resolve([]);
} else {
resolve((response && response.jobs) || []);
}
},
);
});
log('Found ' + jobs.length + ' jobs');
} catch (err) {
log('Error during job extraction:', err.message, err.stack);
}
if (createdNewTab && tab) {
log('Closing created tab...');
chrome.tabs.remove(tab.id).catch(() => {});
}
log(
'fetchJobsViaContentScript: Completed, found ' +
jobs.length +
' unique jobs',
);
return jobs;
} catch (error) {
log('Error in fetchJobsViaContentScript:', error.message, error.stack);
return [];
if (tabs.length === 0) {
tab = await chrome.tabs.create({ url: JOBS_URL, active: false });
createdNewTab = true;
await new Promise((resolve) => setTimeout(resolve, 5000));
} else {
tab = tabs[0];
await chrome.tabs.update(tab.id, { url: JOBS_URL });
await new Promise((resolve) => setTimeout(resolve, 4000));
}
const isLoginPage = await checkIfLoginPage(tab.id);
if (isLoginPage) {
if (isLoggingIn) {
if (createdNewTab) chrome.tabs.remove(tab.id).catch(() => {});
return null;
}
const loginSuccess = await handleAutoLogin(tab);
if (!loginSuccess) {
if (createdNewTab) chrome.tabs.remove(tab.id).catch(() => {});
return null;
}
log('Login successful, waiting 2 minutes before resuming...');
await new Promise((resolve) => setTimeout(resolve, 120000));
await chrome.tabs.update(tab.id, { url: JOBS_URL });
await new Promise((resolve) => setTimeout(resolve, 4000));
}
return { tab, createdNewTab };
}
// Fast path: scrapes whatever profile title is currently active, no switching.
async function fetchJobsFast() {
log('fetchJobsFast: Starting...');
const ready = await getReadyTab();
if (!ready) return [];
const { tab, createdNewTab } = ready;
const jobs = await new Promise((resolve) => {
chrome.tabs.sendMessage(tab.id, { action: 'extractJobs' }, (response) => {
if (chrome.runtime.lastError) resolve([]);
else resolve((response && response.jobs) || []);
});
});
const currentTitle = await getCurrentProfileTitle(tab.id);
jobs.forEach((job) => {
if (job.kind === 'opportunity')
job.professionalTitle = currentTitle || DEFAULT_TITLE;
});
if (createdNewTab && tab) chrome.tabs.remove(tab.id).catch(() => {});
log(
'fetchJobsFast: Completed, ' +
jobs.length +
' jobs under "' +
currentTitle +
'"',
);
return jobs;
}
// Slow path: cycles through every professional title, scraping opportunities
// under each, then resets the profile back to the default title.
async function fetchAllJobsAcrossTitles() {
log('fetchAllJobsAcrossTitles: Starting...');
const ready = await getReadyTab();
if (!ready) return [];
const { tab, createdNewTab } = ready;
let allJobs = [];
for (const title of PROFESSIONAL_TITLES) {
log('--- Title: ' + title + ' ---');
const currentTitle = await getCurrentProfileTitle(tab.id);
if (currentTitle.toLowerCase() !== title.toLowerCase()) {
const switched = await setProfileTitle(tab.id, title);
log(
'Switch to "' +
title +
'": ' +
(switched ? 'ok' : 'FAILED (scraping anyway)'),
);
} else {
log('Already on "' + title + '"');
}
await chrome.tabs.update(tab.id, { url: JOBS_URL });
await new Promise((resolve) => setTimeout(resolve, 4000));
const jobs = await new Promise((resolve) => {
chrome.tabs.sendMessage(tab.id, { action: 'extractJobs' }, (response) => {
if (chrome.runtime.lastError) resolve([]);
else resolve((response && response.jobs) || []);
});
});
jobs.forEach((job) => {
if (job.kind === 'opportunity') job.professionalTitle = title;
allJobs.push(job);
});
log('Collected ' + jobs.length + ' jobs under "' + title + '"');
}
const finalCheck = await getCurrentProfileTitle(tab.id);
if (finalCheck.toLowerCase() !== DEFAULT_TITLE.toLowerCase()) {
log('Resetting profile back to default: ' + DEFAULT_TITLE);
await setProfileTitle(tab.id, DEFAULT_TITLE);
}
if (createdNewTab && tab) chrome.tabs.remove(tab.id).catch(() => {});
log('fetchAllJobsAcrossTitles: Completed, ' + allJobs.length + ' jobs total');
return allJobs;
}
function dedupeJobs(jobs) {
const seen = {};
const deduped = [];
jobs.forEach((job) => {
const key = (job.kind === 'opportunity' ? 'opp:' : 'app:') + job.id;
if (seen[key]) return;
seen[key] = true;
deduped.push(job);
});
return deduped;
}
function sortJobsByRecency(jobs) {
@@ -260,27 +345,30 @@ function sortJobsByRecency(jobs) {
.sort((a, b) => (parseInt(b.id, 10) || 0) - (parseInt(a.id, 10) || 0));
}
function isFrontendJob(title) {
return /front[\s-]?end/i.test(title || '');
function isFrontendJob(job) {
if (job.professionalTitle)
return job.professionalTitle === 'Front-End Engineer';
return /front[\s-]?end/i.test(job.title || '');
}
async function sendNtfyNotification(job, companyName) {
const config = await getConfig();
if (!config.ntfyTopic) {
log('ntfy topic not configured, skipping push notification');
return;
}
if (!config.ntfyTopic) return;
const server = config.ntfyServer.replace(/\/+$/, '');
const headers = { 'Content-Type': 'application/json' };
if (config.ntfyToken) headers.Authorization = 'Bearer ' + config.ntfyToken;
const frontend = isFrontendJob(job.title);
const frontend = isFrontendJob(job);
const roleTag = job.professionalTitle
? ' [' + job.professionalTitle + ']'
: '';
const payload = {
topic: config.ntfyTopic,
title: frontend ? 'Your Perfect Fit - Frontend' : 'New Job on Andela!',
title:
(frontend ? 'Your Perfect Fit - Frontend' : 'New Job on Andela!') +
roleTag,
message: job.title + ' - ' + companyName,
tags: frontend ? ['star', 'computer'] : ['briefcase'],
priority: frontend ? 5 : 4,
@@ -293,20 +381,18 @@ async function sendNtfyNotification(job, companyName) {
headers: headers,
body: JSON.stringify(payload),
});
if (!response.ok) {
if (!response.ok)
log('ntfy push failed: ' + response.status + ' ' + response.statusText);
} else {
log('ntfy push sent for job: ' + job.title);
}
else
log(
'ntfy push sent to topic "' + config.ntfyTopic + '" for: ' + job.title,
);
} catch (error) {
log('ntfy push error:', error.message);
}
}
async function notifyNewJob(job) {
log('Notifying for job: ' + job.title);
await createOffscreen();
chrome.runtime.sendMessage({ action: 'playSound' });
@@ -324,9 +410,6 @@ async function notifyNewJob(job) {
await sendNtfyNotification(job, companyName);
}
// Opens a job's detail page briefly just to read its "Posted X ago" text,
// then closes the tab. Only called for genuinely new jobs, not on every
// 5-minute check, to avoid needlessly opening tabs for jobs we already know about.
async function fetchPostedDate(url) {
if (!url) return null;
let tab;
@@ -338,15 +421,8 @@ async function fetchPostedDate(url) {
tab.id,
{ action: 'extractPostedDate' },
(response) => {
if (chrome.runtime.lastError) {
log(
'Error extracting posted date:',
chrome.runtime.lastError.message,
);
resolve(null);
} else {
resolve((response && response.postedDate) || null);
}
if (chrome.runtime.lastError) resolve(null);
else resolve((response && response.postedDate) || null);
},
);
});
@@ -366,7 +442,6 @@ function startKeepAlive() {
keepAliveInterval = setInterval(() => {
chrome.runtime.getPlatformInfo(() => {
if (chrome.runtime.lastError) {
/* no-op, purpose is just to reset idle timer */
}
});
}, 20000);
@@ -379,95 +454,144 @@ function stopKeepAlive() {
}
}
async function checkForNewJobs() {
log('=== Starting job check ===');
log('Current time:', new Date().toLocaleString());
// Merges freshly scraped jobs into stored jobs, notifies on new opportunities,
// fetches posted dates, tracks a "new" badge window per job, and persists
// everything. Used by both check types.
async function processScrapedJobs(freshJobs) {
const stored = await getStoredData([
'lastAndelaJobs',
'notifiedJobIds',
'postedDates',
'backfillDone',
'newBadgeUntil',
]);
const notifiedJobIds = new Set(stored.notifiedJobIds || []);
const postedDates = stored.postedDates || {};
const backfillDone = stored.backfillDone || false;
const previousJobs = stored.lastAndelaJobs || [];
const newBadgeUntil = stored.newBadgeUntil || {}; // { jobId: expiryTimestampMs }
startKeepAlive();
// Merge: fresh scrape wins for any job id/kind it covers; anything not
// re-scraped this cycle (e.g. other titles during a fast check) is kept
// from the previous snapshot so the popup doesn't lose jobs between sweeps.
const freshKeys = new Set(
freshJobs.map((j) => (j.kind === 'opportunity' ? 'opp:' : 'app:') + j.id),
);
const carriedOver = previousJobs.filter(
(j) => !freshKeys.has((j.kind === 'opportunity' ? 'opp:' : 'app:') + j.id),
);
const merged = dedupeJobs(sortJobsByRecency([...freshJobs, ...carriedOver]));
try {
const allJobs = await fetchJobsViaContentScript();
const sortedJobs = sortJobsByRecency(allJobs);
merged.forEach((job) => {
if (postedDates[job.id]) job.posted_ago = postedDates[job.id];
});
const result = await getStoredData([
'notifiedJobIds',
'postedDates',
'backfillDone',
]);
const notifiedJobIds = new Set(result.notifiedJobIds || []);
const postedDates = result.postedDates || {};
const backfillDone = result.backfillDone || false;
sortedJobs.forEach((job) => {
if (postedDates[job.id]) job.posted_ago = postedDates[job.id];
});
if (!backfillDone) {
const toBackfill = sortedJobs.filter(
(job) => job.kind === 'opportunity' && !postedDates[job.id],
);
log(
'Backfilling posted dates for ' +
toBackfill.length +
' existing jobs...',
);
for (const job of toBackfill) {
const postedDate = await fetchPostedDate(job.url);
if (postedDate) {
job.posted_ago = postedDate;
postedDates[job.id] = postedDate;
}
}
}
const newJobs = sortedJobs.filter(
(job) => job.kind === 'opportunity' && !notifiedJobIds.has(job.id),
if (!backfillDone) {
const toBackfill = merged.filter(
(job) => job.kind === 'opportunity' && !postedDates[job.id],
);
log('Found ' + newJobs.length + ' new jobs');
for (const job of newJobs) {
log(
'Backfilling posted dates for ' + toBackfill.length + ' existing jobs...',
);
for (const job of toBackfill) {
const postedDate = await fetchPostedDate(job.url);
if (postedDate) {
job.posted_ago = postedDate;
postedDates[job.id] = postedDate;
}
await notifyNewJob(job);
notifiedJobIds.add(job.id);
}
await setStoredData({
notifiedJobIds: Array.from(notifiedJobIds),
postedDates: postedDates,
backfillDone: true,
lastAndelaJobs: sortedJobs,
lastAndelaUpdated: new Date().toISOString(),
totalAndelaMatching: sortedJobs.filter((j) => j.kind === 'opportunity')
.length,
});
log('Notifications stored successfully');
} catch (error) {
console.error('Error in checkForNewJobs:', error);
log('Error details:', error.message, error.stack);
} finally {
stopKeepAlive();
}
log('=== Job check completed ===');
const newJobs = merged.filter(
(job) => job.kind === 'opportunity' && !notifiedJobIds.has(job.id),
);
log('Found ' + newJobs.length + ' new jobs');
const now = Date.now();
const badgeDurationMs = MULTI_TITLE_INTERVAL * 60 * 1000;
for (const job of newJobs) {
const postedDate = await fetchPostedDate(job.url);
if (postedDate) {
job.posted_ago = postedDate;
postedDates[job.id] = postedDate;
}
await notifyNewJob(job);
notifiedJobIds.add(job.id);
newBadgeUntil[job.id] = now + badgeDurationMs;
}
// Clean up expired badges so storage doesn't grow forever.
Object.keys(newBadgeUntil).forEach((id) => {
if (newBadgeUntil[id] <= now) delete newBadgeUntil[id];
});
await setStoredData({
notifiedJobIds: Array.from(notifiedJobIds),
postedDates: postedDates,
backfillDone: true,
newBadgeUntil: newBadgeUntil,
lastAndelaJobs: merged,
lastAndelaUpdated: new Date().toISOString(),
totalAndelaMatching: merged.filter((j) => j.kind === 'opportunity').length,
});
}
chrome.alarms.create('checkAndelaJobs', { periodInMinutes: CHECK_INTERVAL });
async function checkForNewJobsFast() {
if (activeOperation) {
log('"' + activeOperation + '" in progress, skipping fast check');
return;
}
activeOperation = 'fast';
log('=== Starting fast check ===');
startKeepAlive();
try {
const freshJobs = await fetchJobsFast();
await processScrapedJobs(freshJobs);
} catch (error) {
console.error('Error in checkForNewJobsFast:', error);
} finally {
stopKeepAlive();
activeOperation = null;
}
log('=== Fast check completed ===');
}
async function runMultiTitleSweep() {
if (activeOperation) {
log('"' + activeOperation + '" in progress, skipping sweep');
return;
}
activeOperation = 'sweep';
log('=== Starting multi-title sweep ===');
startKeepAlive();
try {
const freshJobs = await fetchAllJobsAcrossTitles();
await processScrapedJobs(freshJobs);
} catch (error) {
console.error('Error in runMultiTitleSweep:', error);
} finally {
stopKeepAlive();
activeOperation = null;
}
log('=== Multi-title sweep completed ===');
}
chrome.alarms.create('fastCheck', { periodInMinutes: FAST_CHECK_INTERVAL });
chrome.alarms.create('multiTitleSweep', {
periodInMinutes: MULTI_TITLE_INTERVAL,
});
chrome.alarms.onAlarm.addListener((alarm) => {
if (alarm.name === 'checkAndelaJobs') checkForNewJobs();
if (alarm.name === 'fastCheck') checkForNewJobsFast();
if (alarm.name === 'multiTitleSweep') runMultiTitleSweep();
});
chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
log('Received message:', request);
if (request.action === 'checkNow') {
log('Manual check requested from popup');
checkForNewJobs();
checkForNewJobsFast();
sendResponse({ status: 'Check initiated' });
}
return true;
@@ -475,14 +599,12 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
createOffscreen().catch(console.error);
chrome.runtime.onInstalled.addListener(() => {
log('Extension installed/updated, checking for jobs...');
checkForNewJobs();
chrome.runtime.onInstalled.addListener(async () => {
await runMultiTitleSweep(); // covers Full-Stack too, so no separate fast check needed right after
testNotification();
});
async function testNotification() {
log('Testing notification system...');
chrome.notifications.create('test-notification', {
type: 'basic',
iconUrl: NOTIFICATION_ICON,
@@ -492,19 +614,14 @@ async function testNotification() {
});
}
log('Andela Job Notifier started at:', new Date().toLocaleString());
chrome.alarms.get('checkAndelaJobs', (alarm) => {
if (alarm) {
log(
'Alarm is set, next check at:',
new Date(alarm.scheduledTime).toLocaleString(),
);
} else {
log('No alarm found, creating one...');
chrome.alarms.create('checkAndelaJobs', {
periodInMinutes: CHECK_INTERVAL,
delayInMinutes: 0.1,
});
}
['fastCheck', 'multiTitleSweep'].forEach((name) => {
chrome.alarms.get(name, (alarm) => {
if (!alarm) {
chrome.alarms.create(name, {
periodInMinutes:
name === 'fastCheck' ? FAST_CHECK_INTERVAL : MULTI_TITLE_INTERVAL,
delayInMinutes: 0.1,
});
}
});
});