Fixed multi-job scrape, and added labels and single job scrape
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
// content.js - Handles job extraction and login automation for app.andela.com
|
||||
// content.js - Handles job extraction, login automation, and profile
|
||||
// title switching for app.andela.com
|
||||
|
||||
console.log('[Andela Content Script] Loaded on', window.location.href);
|
||||
|
||||
@@ -7,25 +8,13 @@ function isLoginPage() {
|
||||
var emailLabel = document.querySelector('label[for="email"]');
|
||||
var hasLoginForm = !!(emailInput && emailLabel);
|
||||
var isLoginUrl = window.location.pathname.indexOf('/login') === 0;
|
||||
|
||||
console.log(
|
||||
'[Andela Content Script] Login check - hasForm:',
|
||||
hasLoginForm,
|
||||
'isLoginUrl:',
|
||||
isLoginUrl,
|
||||
);
|
||||
return hasLoginForm || (isLoginUrl && !!emailInput);
|
||||
}
|
||||
|
||||
function enterEmailAndClickLogin(email) {
|
||||
try {
|
||||
console.log('[Andela Content Script] Attempting to enter email:', email);
|
||||
|
||||
var emailInput = document.querySelector('input[type="email"]#email');
|
||||
if (!emailInput) {
|
||||
console.log('[Andela Content Script] Email input not found');
|
||||
return false;
|
||||
}
|
||||
if (!emailInput) return false;
|
||||
|
||||
emailInput.focus();
|
||||
emailInput.value = email;
|
||||
@@ -33,39 +22,26 @@ function enterEmailAndClickLogin(email) {
|
||||
emailInput.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
emailInput.dispatchEvent(new KeyboardEvent('keyup', { bubbles: true }));
|
||||
|
||||
console.log(
|
||||
'[Andela Content Script] Email entered, looking for submit button...',
|
||||
);
|
||||
|
||||
setTimeout(function () {
|
||||
var submitButton = document.querySelector('button[type="submit"]');
|
||||
|
||||
if (!submitButton) {
|
||||
var buttons = document.querySelectorAll('button');
|
||||
for (var i = 0; i < buttons.length; i++) {
|
||||
var btnText = buttons[i].textContent.toLowerCase();
|
||||
var t = buttons[i].textContent.toLowerCase();
|
||||
if (
|
||||
btnText.indexOf('login') !== -1 ||
|
||||
btnText.indexOf('sign in') !== -1 ||
|
||||
btnText.indexOf('continue') !== -1 ||
|
||||
btnText.indexOf('submit') !== -1
|
||||
t.indexOf('login') !== -1 ||
|
||||
t.indexOf('sign in') !== -1 ||
|
||||
t.indexOf('continue') !== -1 ||
|
||||
t.indexOf('submit') !== -1
|
||||
) {
|
||||
submitButton = buttons[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (submitButton) {
|
||||
console.log(
|
||||
'[Andela Content Script] Clicking submit button:',
|
||||
submitButton.textContent,
|
||||
);
|
||||
submitButton.click();
|
||||
} else {
|
||||
console.log(
|
||||
'[Andela Content Script] Submit button not found, trying form submit',
|
||||
);
|
||||
var form = emailInput.closest('form');
|
||||
if (form) {
|
||||
if (form.requestSubmit) form.requestSubmit();
|
||||
@@ -89,15 +65,13 @@ function isOpportunitySection(headingText) {
|
||||
return /opportunit/i.test(headingText || '');
|
||||
}
|
||||
|
||||
// Walks the page in DOM order and tags every "See more" button and every
|
||||
// job-posting link with the heading text of the section it falls under.
|
||||
function getPageSections() {
|
||||
var elements = document.querySelectorAll(
|
||||
'h3, button, a[href*="/job-postings/"]',
|
||||
);
|
||||
var currentHeading = '';
|
||||
var seeMoreButtons = []; // { button, heading }
|
||||
var jobLinks = []; // { link, heading }
|
||||
var seeMoreButtons = [];
|
||||
var jobLinks = [];
|
||||
|
||||
elements.forEach(function (el) {
|
||||
if (el.tagName === 'H3') {
|
||||
@@ -123,8 +97,6 @@ function getPageSections() {
|
||||
return { seeMoreButtons: seeMoreButtons, jobLinks: jobLinks };
|
||||
}
|
||||
|
||||
// Only expands "See more" under opportunity sections — leaves the
|
||||
// applied-jobs section collapsed as-is.
|
||||
function expandAllSeeMoreButtons(maxRounds) {
|
||||
maxRounds = maxRounds || 25;
|
||||
return new Promise(function (resolve) {
|
||||
@@ -134,9 +106,7 @@ function expandAllSeeMoreButtons(maxRounds) {
|
||||
var sections = getPageSections();
|
||||
for (var i = 0; i < sections.seeMoreButtons.length; i++) {
|
||||
var entry = sections.seeMoreButtons[i];
|
||||
if (isOpportunitySection(entry.heading)) {
|
||||
return entry.button;
|
||||
}
|
||||
if (isOpportunitySection(entry.heading)) return entry.button;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -145,15 +115,9 @@ function expandAllSeeMoreButtons(maxRounds) {
|
||||
rounds++;
|
||||
var btn = findSeeMoreButton();
|
||||
if (!btn || rounds > maxRounds) {
|
||||
console.log(
|
||||
'[Andela Content Script] Done expanding opportunity "See more" buttons after',
|
||||
rounds - 1,
|
||||
'click(s)',
|
||||
);
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
console.log('[Andela Content Script] Clicking:', btn.textContent.trim());
|
||||
btn.click();
|
||||
setTimeout(clickNext, 700);
|
||||
}
|
||||
@@ -199,9 +163,7 @@ function extractJobFromLink(link) {
|
||||
});
|
||||
|
||||
var url = href;
|
||||
if (url && url.indexOf('http') !== 0) {
|
||||
url = window.location.origin + url;
|
||||
}
|
||||
if (url && url.indexOf('http') !== 0) url = window.location.origin + url;
|
||||
|
||||
return {
|
||||
id: id,
|
||||
@@ -215,13 +177,9 @@ function extractJobFromLink(link) {
|
||||
}
|
||||
|
||||
function extractJobsFromPage() {
|
||||
console.log('[Andela Content Script] Extracting jobs...');
|
||||
var jobs = [];
|
||||
var seen = {};
|
||||
var jobLinks = getPageSections().jobLinks;
|
||||
console.log(
|
||||
'[Andela Content Script] Found ' + jobLinks.length + ' job-posting links',
|
||||
);
|
||||
|
||||
jobLinks.forEach(function (entry) {
|
||||
try {
|
||||
@@ -232,16 +190,7 @@ function extractJobsFromPage() {
|
||||
job.kind = isOpportunitySection(entry.heading)
|
||||
? 'opportunity'
|
||||
: 'applied';
|
||||
if (job.title) {
|
||||
jobs.push(job);
|
||||
console.log(
|
||||
'[Andela Content Script] Extracted job:',
|
||||
job.title,
|
||||
'(' + job.id + ')',
|
||||
'-',
|
||||
job.kind,
|
||||
);
|
||||
}
|
||||
if (job.title) jobs.push(job);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
'[Andela Content Script] Error extracting a job card:',
|
||||
@@ -250,21 +199,148 @@ function extractJobsFromPage() {
|
||||
}
|
||||
});
|
||||
|
||||
console.log(
|
||||
'[Andela Content Script] Extracted ' + jobs.length + ' jobs total',
|
||||
);
|
||||
return jobs;
|
||||
}
|
||||
|
||||
chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
|
||||
console.log('[Andela Content Script] Received message:', request);
|
||||
// ============================================
|
||||
// Profile title switching (used to surface jobs for other roles)
|
||||
// ============================================
|
||||
|
||||
function getCurrentProfileTitle() {
|
||||
var el = document.querySelector('.text-primary-600.mt-0\\.5.text-sm');
|
||||
return el ? el.textContent.trim() : '';
|
||||
}
|
||||
|
||||
function waitFor(conditionFn, timeoutMs, intervalMs) {
|
||||
timeoutMs = timeoutMs || 5000;
|
||||
intervalMs = intervalMs || 200;
|
||||
return new Promise(function (resolve) {
|
||||
var elapsed = 0;
|
||||
function check() {
|
||||
var result = conditionFn();
|
||||
if (result) {
|
||||
resolve(result);
|
||||
return;
|
||||
}
|
||||
elapsed += intervalMs;
|
||||
if (elapsed >= timeoutMs) {
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
setTimeout(check, intervalMs);
|
||||
}
|
||||
check();
|
||||
});
|
||||
}
|
||||
|
||||
// Plain .click() doesn't reliably open this dropdown when dispatched
|
||||
// programmatically — needs the full pointer-event sequence instead.
|
||||
function dispatchFullClick(el) {
|
||||
el.focus();
|
||||
['pointerdown', 'mousedown', 'pointerup', 'mouseup', 'click'].forEach(
|
||||
function (type) {
|
||||
el.dispatchEvent(
|
||||
new MouseEvent(type, { bubbles: true, cancelable: true, view: window }),
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function setProfessionalTitle(title) {
|
||||
console.log(
|
||||
'[Andela Content Script] setProfessionalTitle: looking for edit button...',
|
||||
);
|
||||
var editBtn = await waitFor(function () {
|
||||
return document.querySelector('button[aria-label="Edit profile details"]');
|
||||
}, 6000);
|
||||
if (!editBtn) {
|
||||
console.log('[Andela Content Script] STEP FAILED: edit button not found');
|
||||
return false;
|
||||
}
|
||||
console.log(
|
||||
'[Andela Content Script] STEP OK: edit button found, clicking...',
|
||||
);
|
||||
editBtn.click();
|
||||
|
||||
console.log(
|
||||
'[Andela Content Script] looking for professional_title input...',
|
||||
);
|
||||
var titleInput = await waitFor(function () {
|
||||
return document.querySelector('#professional_title');
|
||||
}, 5000);
|
||||
if (!titleInput) {
|
||||
console.log(
|
||||
'[Andela Content Script] STEP FAILED: professional_title input not found (modal may not have opened)',
|
||||
);
|
||||
return false;
|
||||
}
|
||||
console.log(
|
||||
'[Andela Content Script] STEP OK: input found, opening dropdown...',
|
||||
);
|
||||
dispatchFullClick(titleInput);
|
||||
|
||||
console.log('[Andela Content Script] looking for option matching:', title);
|
||||
var optionEl = await waitFor(
|
||||
function () {
|
||||
var options = document.querySelectorAll('[role="option"]');
|
||||
for (var i = 0; i < options.length; i++) {
|
||||
var span = options[i].querySelector('span.col-start-2') || options[i];
|
||||
if (
|
||||
span &&
|
||||
span.textContent.trim().toLowerCase() === title.toLowerCase()
|
||||
) {
|
||||
return options[i];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
},
|
||||
5000,
|
||||
400,
|
||||
);
|
||||
if (!optionEl) {
|
||||
console.log(
|
||||
'[Andela Content Script] STEP FAILED: no option matched "' + title + '"',
|
||||
);
|
||||
return false;
|
||||
}
|
||||
console.log('[Andela Content Script] STEP OK: option found, clicking...');
|
||||
optionEl.click();
|
||||
|
||||
await new Promise(function (r) {
|
||||
setTimeout(r, 400);
|
||||
});
|
||||
|
||||
console.log('[Andela Content Script] looking for save button...');
|
||||
var saveBtn = document.querySelector(
|
||||
'button[type="submit"][form="edit-profile-details-form"]',
|
||||
);
|
||||
if (!saveBtn) {
|
||||
console.log('[Andela Content Script] STEP FAILED: save button not found');
|
||||
return false;
|
||||
}
|
||||
console.log(
|
||||
'[Andela Content Script] STEP OK: save button found, clicking...',
|
||||
);
|
||||
saveBtn.click();
|
||||
|
||||
console.log('[Andela Content Script] waiting for modal to close...');
|
||||
var closed = await waitFor(function () {
|
||||
return !document.querySelector('#professional_title');
|
||||
}, 15000);
|
||||
|
||||
console.log(
|
||||
'[Andela Content Script] setProfessionalTitle(' + title + ') closed modal:',
|
||||
!!closed,
|
||||
);
|
||||
return !!closed;
|
||||
}
|
||||
|
||||
chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
|
||||
if (request.action === 'extractJobs') {
|
||||
expandAllSeeMoreButtons().then(function () {
|
||||
setTimeout(function () {
|
||||
try {
|
||||
var jobs = extractJobsFromPage();
|
||||
sendResponse({ jobs: jobs });
|
||||
sendResponse({ jobs: extractJobsFromPage() });
|
||||
} catch (error) {
|
||||
console.error('[Andela Content Script] Error:', error);
|
||||
sendResponse({ jobs: [] });
|
||||
@@ -275,24 +351,31 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
|
||||
}
|
||||
|
||||
if (request.action === 'checkLoginPage') {
|
||||
var result = isLoginPage();
|
||||
console.log('[Andela Content Script] Is login page:', result);
|
||||
sendResponse({ isLoginPage: result });
|
||||
sendResponse({ isLoginPage: isLoginPage() });
|
||||
return true;
|
||||
}
|
||||
|
||||
if (request.action === 'enterEmailAndLogin') {
|
||||
var success = enterEmailAndClickLogin(request.email);
|
||||
sendResponse({ success: success });
|
||||
sendResponse({ success: enterEmailAndClickLogin(request.email) });
|
||||
return true;
|
||||
}
|
||||
|
||||
if (request.action === 'extractPostedDate') {
|
||||
var el = document.querySelector('span.text-success-500');
|
||||
var postedText = el ? el.textContent.trim() : '';
|
||||
console.log(
|
||||
'[Andela Content Script] Extracted posted date text:',
|
||||
postedText,
|
||||
);
|
||||
sendResponse({ postedDate: postedText });
|
||||
sendResponse({ postedDate: el ? el.textContent.trim() : '' });
|
||||
return true;
|
||||
}
|
||||
|
||||
if (request.action === 'getCurrentProfileTitle') {
|
||||
sendResponse({ title: getCurrentProfileTitle() });
|
||||
return true;
|
||||
}
|
||||
|
||||
if (request.action === 'setProfessionalTitle') {
|
||||
setProfessionalTitle(request.title).then(function (success) {
|
||||
sendResponse({ success: success });
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
Reference in New Issue
Block a user