Removed all job category aggregation, display dates job posted and minor fixes
This commit is contained in:
@@ -435,6 +435,44 @@ async function fetchPostedDate(url) {
|
||||
}
|
||||
}
|
||||
|
||||
// Converts Andela's relative "Posted X ago" text into an absolute timestamp
|
||||
// (ms since epoch), computed once at fetch time. Storing the absolute time
|
||||
// instead of the display string lets the popup recompute "X ago" live,
|
||||
// so it never goes stale.
|
||||
//
|
||||
// Andela uses a few different phrasings depending on how recent the post
|
||||
// is: "Posted 10 hours ago", "Posted yesterday", "Posted today", etc. —
|
||||
// not all of them have a number + unit + "ago", so those need explicit
|
||||
// handling rather than relying purely on the regex.
|
||||
function parsePostedAgoToTimestamp(text) {
|
||||
if (!text) return null;
|
||||
var cleaned = text.trim().toLowerCase();
|
||||
|
||||
if (/posted\s+today/.test(cleaned) || /posted\s+just now/.test(cleaned)) {
|
||||
return Date.now();
|
||||
}
|
||||
if (/posted\s+yesterday/.test(cleaned)) {
|
||||
return Date.now() - 24 * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
var match = cleaned.match(
|
||||
/posted\s+(\d+)\s+(minute|hour|day|week|month)s?\s+ago/i,
|
||||
);
|
||||
if (!match) return null;
|
||||
|
||||
var amount = parseInt(match[1], 10);
|
||||
var unit = match[2].toLowerCase();
|
||||
var msPerUnit = {
|
||||
minute: 60 * 1000,
|
||||
hour: 60 * 60 * 1000,
|
||||
day: 24 * 60 * 60 * 1000,
|
||||
week: 7 * 24 * 60 * 60 * 1000,
|
||||
month: 30 * 24 * 60 * 60 * 1000,
|
||||
};
|
||||
|
||||
return Date.now() - amount * msPerUnit[unit];
|
||||
}
|
||||
|
||||
let keepAliveInterval = null;
|
||||
|
||||
function startKeepAlive() {
|
||||
@@ -455,21 +493,21 @@ function stopKeepAlive() {
|
||||
}
|
||||
|
||||
// Merges freshly scraped jobs into stored jobs, notifies on new opportunities,
|
||||
// fetches posted dates, tracks a "new" badge window per job, and persists
|
||||
// everything. Used by both check types.
|
||||
// resolves posted-date timestamps, tracks a "new" badge window per job, and
|
||||
// persists everything. Used by both check types.
|
||||
async function processScrapedJobs(freshJobs) {
|
||||
const stored = await getStoredData([
|
||||
'lastAndelaJobs',
|
||||
'notifiedJobIds',
|
||||
'postedDates',
|
||||
'postedTimestamps',
|
||||
'backfillDone',
|
||||
'newBadgeUntil',
|
||||
]);
|
||||
const notifiedJobIds = new Set(stored.notifiedJobIds || []);
|
||||
const postedDates = stored.postedDates || {};
|
||||
const postedTimestamps = stored.postedTimestamps || {}; // { jobId: ms since epoch }
|
||||
const backfillDone = stored.backfillDone || false;
|
||||
const previousJobs = stored.lastAndelaJobs || [];
|
||||
const newBadgeUntil = stored.newBadgeUntil || {}; // { jobId: expiryTimestampMs }
|
||||
const newBadgeUntil = stored.newBadgeUntil || {};
|
||||
|
||||
// Merge: fresh scrape wins for any job id/kind it covers; anything not
|
||||
// re-scraped this cycle (e.g. other titles during a fast check) is kept
|
||||
@@ -483,21 +521,31 @@ async function processScrapedJobs(freshJobs) {
|
||||
const merged = dedupeJobs(sortJobsByRecency([...freshJobs, ...carriedOver]));
|
||||
|
||||
merged.forEach((job) => {
|
||||
if (postedDates[job.id]) job.posted_ago = postedDates[job.id];
|
||||
if (postedTimestamps[job.id])
|
||||
job.posted_timestamp = postedTimestamps[job.id];
|
||||
});
|
||||
|
||||
if (!backfillDone) {
|
||||
const toBackfill = merged.filter(
|
||||
(job) => job.kind === 'opportunity' && !postedDates[job.id],
|
||||
(job) => job.kind === 'opportunity' && !postedTimestamps[job.id],
|
||||
);
|
||||
log(
|
||||
'Backfilling posted dates for ' + toBackfill.length + ' existing jobs...',
|
||||
);
|
||||
for (const job of toBackfill) {
|
||||
const postedDate = await fetchPostedDate(job.url);
|
||||
if (postedDate) {
|
||||
job.posted_ago = postedDate;
|
||||
postedDates[job.id] = postedDate;
|
||||
const postedText = await fetchPostedDate(job.url);
|
||||
const timestamp = parsePostedAgoToTimestamp(postedText);
|
||||
if (timestamp) {
|
||||
job.posted_timestamp = timestamp;
|
||||
postedTimestamps[job.id] = timestamp;
|
||||
} else {
|
||||
log(
|
||||
'Could not parse posted date for job ' +
|
||||
job.id +
|
||||
': "' +
|
||||
postedText +
|
||||
'"',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -511,10 +559,19 @@ async function processScrapedJobs(freshJobs) {
|
||||
const badgeDurationMs = MULTI_TITLE_INTERVAL * 60 * 1000;
|
||||
|
||||
for (const job of newJobs) {
|
||||
const postedDate = await fetchPostedDate(job.url);
|
||||
if (postedDate) {
|
||||
job.posted_ago = postedDate;
|
||||
postedDates[job.id] = postedDate;
|
||||
const postedText = await fetchPostedDate(job.url);
|
||||
const timestamp = parsePostedAgoToTimestamp(postedText);
|
||||
if (timestamp) {
|
||||
job.posted_timestamp = timestamp;
|
||||
postedTimestamps[job.id] = timestamp;
|
||||
} else {
|
||||
log(
|
||||
'Could not parse posted date for job ' +
|
||||
job.id +
|
||||
': "' +
|
||||
postedText +
|
||||
'"',
|
||||
);
|
||||
}
|
||||
await notifyNewJob(job);
|
||||
notifiedJobIds.add(job.id);
|
||||
@@ -528,7 +585,7 @@ async function processScrapedJobs(freshJobs) {
|
||||
|
||||
await setStoredData({
|
||||
notifiedJobIds: Array.from(notifiedJobIds),
|
||||
postedDates: postedDates,
|
||||
postedTimestamps: postedTimestamps,
|
||||
backfillDone: true,
|
||||
newBadgeUntil: newBadgeUntil,
|
||||
lastAndelaJobs: merged,
|
||||
|
||||
Reference in New Issue
Block a user